LLVM 19.0.0git
InstCombineCompares.cpp
Go to the documentation of this file.
1//===- InstCombineCompares.cpp --------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitICmp and visitFCmp functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/ADT/APSInt.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/Statistic.h"
25#include "llvm/IR/DataLayout.h"
30#include <bitset>
31
32using namespace llvm;
33using namespace PatternMatch;
34
35#define DEBUG_TYPE "instcombine"
36
37// How many times is a select replaced by one of its operands?
38STATISTIC(NumSel, "Number of select opts");
39
40
41/// Compute Result = In1+In2, returning true if the result overflowed for this
42/// type.
43static bool addWithOverflow(APInt &Result, const APInt &In1,
44 const APInt &In2, bool IsSigned = false) {
45 bool Overflow;
46 if (IsSigned)
47 Result = In1.sadd_ov(In2, Overflow);
48 else
49 Result = In1.uadd_ov(In2, Overflow);
50
51 return Overflow;
52}
53
54/// Compute Result = In1-In2, returning true if the result overflowed for this
55/// type.
56static bool subWithOverflow(APInt &Result, const APInt &In1,
57 const APInt &In2, bool IsSigned = false) {
58 bool Overflow;
59 if (IsSigned)
60 Result = In1.ssub_ov(In2, Overflow);
61 else
62 Result = In1.usub_ov(In2, Overflow);
63
64 return Overflow;
65}
66
67/// Given an icmp instruction, return true if any use of this comparison is a
68/// branch on sign bit comparison.
69static bool hasBranchUse(ICmpInst &I) {
70 for (auto *U : I.users())
71 if (isa<BranchInst>(U))
72 return true;
73 return false;
74}
75
76/// Returns true if the exploded icmp can be expressed as a signed comparison
77/// to zero and updates the predicate accordingly.
78/// The signedness of the comparison is preserved.
79/// TODO: Refactor with decomposeBitTestICmp()?
80static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) {
81 if (!ICmpInst::isSigned(Pred))
82 return false;
83
84 if (C.isZero())
85 return ICmpInst::isRelational(Pred);
86
87 if (C.isOne()) {
88 if (Pred == ICmpInst::ICMP_SLT) {
89 Pred = ICmpInst::ICMP_SLE;
90 return true;
91 }
92 } else if (C.isAllOnes()) {
93 if (Pred == ICmpInst::ICMP_SGT) {
94 Pred = ICmpInst::ICMP_SGE;
95 return true;
96 }
97 }
98
99 return false;
100}
101
102/// This is called when we see this pattern:
103/// cmp pred (load (gep GV, ...)), cmpcst
104/// where GV is a global variable with a constant initializer. Try to simplify
105/// this into some simple computation that does not need the load. For example
106/// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3".
107///
108/// If AndCst is non-null, then the loaded value is masked with that constant
109/// before doing the comparison. This handles cases like "A[i]&4 == 0".
112 ConstantInt *AndCst) {
113 if (LI->isVolatile() || LI->getType() != GEP->getResultElementType() ||
114 GV->getValueType() != GEP->getSourceElementType() || !GV->isConstant() ||
116 return nullptr;
117
119 if (!isa<ConstantArray>(Init) && !isa<ConstantDataArray>(Init))
120 return nullptr;
121
122 uint64_t ArrayElementCount = Init->getType()->getArrayNumElements();
123 // Don't blow up on huge arrays.
124 if (ArrayElementCount > MaxArraySizeForCombine)
125 return nullptr;
126
127 // There are many forms of this optimization we can handle, for now, just do
128 // the simple index into a single-dimensional array.
129 //
130 // Require: GEP GV, 0, i {{, constant indices}}
131 if (GEP->getNumOperands() < 3 || !isa<ConstantInt>(GEP->getOperand(1)) ||
132 !cast<ConstantInt>(GEP->getOperand(1))->isZero() ||
133 isa<Constant>(GEP->getOperand(2)))
134 return nullptr;
135
136 // Check that indices after the variable are constants and in-range for the
137 // type they index. Collect the indices. This is typically for arrays of
138 // structs.
139 SmallVector<unsigned, 4> LaterIndices;
140
141 Type *EltTy = Init->getType()->getArrayElementType();
142 for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) {
143 ConstantInt *Idx = dyn_cast<ConstantInt>(GEP->getOperand(i));
144 if (!Idx)
145 return nullptr; // Variable index.
146
147 uint64_t IdxVal = Idx->getZExtValue();
148 if ((unsigned)IdxVal != IdxVal)
149 return nullptr; // Too large array index.
150
151 if (StructType *STy = dyn_cast<StructType>(EltTy))
152 EltTy = STy->getElementType(IdxVal);
153 else if (ArrayType *ATy = dyn_cast<ArrayType>(EltTy)) {
154 if (IdxVal >= ATy->getNumElements())
155 return nullptr;
156 EltTy = ATy->getElementType();
157 } else {
158 return nullptr; // Unknown type.
159 }
160
161 LaterIndices.push_back(IdxVal);
162 }
163
164 enum { Overdefined = -3, Undefined = -2 };
165
166 // Variables for our state machines.
167
168 // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form
169 // "i == 47 | i == 87", where 47 is the first index the condition is true for,
170 // and 87 is the second (and last) index. FirstTrueElement is -2 when
171 // undefined, otherwise set to the first true element. SecondTrueElement is
172 // -2 when undefined, -3 when overdefined and >= 0 when that index is true.
173 int FirstTrueElement = Undefined, SecondTrueElement = Undefined;
174
175 // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the
176 // form "i != 47 & i != 87". Same state transitions as for true elements.
177 int FirstFalseElement = Undefined, SecondFalseElement = Undefined;
178
179 /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these
180 /// define a state machine that triggers for ranges of values that the index
181 /// is true or false for. This triggers on things like "abbbbc"[i] == 'b'.
182 /// This is -2 when undefined, -3 when overdefined, and otherwise the last
183 /// index in the range (inclusive). We use -2 for undefined here because we
184 /// use relative comparisons and don't want 0-1 to match -1.
185 int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined;
186
187 // MagicBitvector - This is a magic bitvector where we set a bit if the
188 // comparison is true for element 'i'. If there are 64 elements or less in
189 // the array, this will fully represent all the comparison results.
190 uint64_t MagicBitvector = 0;
191
192 // Scan the array and see if one of our patterns matches.
193 Constant *CompareRHS = cast<Constant>(ICI.getOperand(1));
194 for (unsigned i = 0, e = ArrayElementCount; i != e; ++i) {
195 Constant *Elt = Init->getAggregateElement(i);
196 if (!Elt)
197 return nullptr;
198
199 // If this is indexing an array of structures, get the structure element.
200 if (!LaterIndices.empty()) {
201 Elt = ConstantFoldExtractValueInstruction(Elt, LaterIndices);
202 if (!Elt)
203 return nullptr;
204 }
205
206 // If the element is masked, handle it.
207 if (AndCst) {
208 Elt = ConstantFoldBinaryOpOperands(Instruction::And, Elt, AndCst, DL);
209 if (!Elt)
210 return nullptr;
211 }
212
213 // Find out if the comparison would be true or false for the i'th element.
215 CompareRHS, DL, &TLI);
216 // If the result is undef for this element, ignore it.
217 if (isa<UndefValue>(C)) {
218 // Extend range state machines to cover this element in case there is an
219 // undef in the middle of the range.
220 if (TrueRangeEnd == (int)i - 1)
221 TrueRangeEnd = i;
222 if (FalseRangeEnd == (int)i - 1)
223 FalseRangeEnd = i;
224 continue;
225 }
226
227 // If we can't compute the result for any of the elements, we have to give
228 // up evaluating the entire conditional.
229 if (!isa<ConstantInt>(C))
230 return nullptr;
231
232 // Otherwise, we know if the comparison is true or false for this element,
233 // update our state machines.
234 bool IsTrueForElt = !cast<ConstantInt>(C)->isZero();
235
236 // State machine for single/double/range index comparison.
237 if (IsTrueForElt) {
238 // Update the TrueElement state machine.
239 if (FirstTrueElement == Undefined)
240 FirstTrueElement = TrueRangeEnd = i; // First true element.
241 else {
242 // Update double-compare state machine.
243 if (SecondTrueElement == Undefined)
244 SecondTrueElement = i;
245 else
246 SecondTrueElement = Overdefined;
247
248 // Update range state machine.
249 if (TrueRangeEnd == (int)i - 1)
250 TrueRangeEnd = i;
251 else
252 TrueRangeEnd = Overdefined;
253 }
254 } else {
255 // Update the FalseElement state machine.
256 if (FirstFalseElement == Undefined)
257 FirstFalseElement = FalseRangeEnd = i; // First false element.
258 else {
259 // Update double-compare state machine.
260 if (SecondFalseElement == Undefined)
261 SecondFalseElement = i;
262 else
263 SecondFalseElement = Overdefined;
264
265 // Update range state machine.
266 if (FalseRangeEnd == (int)i - 1)
267 FalseRangeEnd = i;
268 else
269 FalseRangeEnd = Overdefined;
270 }
271 }
272
273 // If this element is in range, update our magic bitvector.
274 if (i < 64 && IsTrueForElt)
275 MagicBitvector |= 1ULL << i;
276
277 // If all of our states become overdefined, bail out early. Since the
278 // predicate is expensive, only check it every 8 elements. This is only
279 // really useful for really huge arrays.
280 if ((i & 8) == 0 && i >= 64 && SecondTrueElement == Overdefined &&
281 SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined &&
282 FalseRangeEnd == Overdefined)
283 return nullptr;
284 }
285
286 // Now that we've scanned the entire array, emit our new comparison(s). We
287 // order the state machines in complexity of the generated code.
288 Value *Idx = GEP->getOperand(2);
289
290 // If the index is larger than the pointer offset size of the target, truncate
291 // the index down like the GEP would do implicitly. We don't have to do this
292 // for an inbounds GEP because the index can't be out of range.
293 if (!GEP->isInBounds()) {
294 Type *PtrIdxTy = DL.getIndexType(GEP->getType());
295 unsigned OffsetSize = PtrIdxTy->getIntegerBitWidth();
296 if (Idx->getType()->getPrimitiveSizeInBits().getFixedValue() > OffsetSize)
297 Idx = Builder.CreateTrunc(Idx, PtrIdxTy);
298 }
299
300 // If inbounds keyword is not present, Idx * ElementSize can overflow.
301 // Let's assume that ElementSize is 2 and the wanted value is at offset 0.
302 // Then, there are two possible values for Idx to match offset 0:
303 // 0x00..00, 0x80..00.
304 // Emitting 'icmp eq Idx, 0' isn't correct in this case because the
305 // comparison is false if Idx was 0x80..00.
306 // We need to erase the highest countTrailingZeros(ElementSize) bits of Idx.
307 unsigned ElementSize =
308 DL.getTypeAllocSize(Init->getType()->getArrayElementType());
309 auto MaskIdx = [&](Value *Idx) {
310 if (!GEP->isInBounds() && llvm::countr_zero(ElementSize) != 0) {
311 Value *Mask = ConstantInt::get(Idx->getType(), -1);
312 Mask = Builder.CreateLShr(Mask, llvm::countr_zero(ElementSize));
313 Idx = Builder.CreateAnd(Idx, Mask);
314 }
315 return Idx;
316 };
317
318 // If the comparison is only true for one or two elements, emit direct
319 // comparisons.
320 if (SecondTrueElement != Overdefined) {
321 Idx = MaskIdx(Idx);
322 // None true -> false.
323 if (FirstTrueElement == Undefined)
324 return replaceInstUsesWith(ICI, Builder.getFalse());
325
326 Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement);
327
328 // True for one element -> 'i == 47'.
329 if (SecondTrueElement == Undefined)
330 return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx);
331
332 // True for two elements -> 'i == 47 | i == 72'.
333 Value *C1 = Builder.CreateICmpEQ(Idx, FirstTrueIdx);
334 Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement);
335 Value *C2 = Builder.CreateICmpEQ(Idx, SecondTrueIdx);
336 return BinaryOperator::CreateOr(C1, C2);
337 }
338
339 // If the comparison is only false for one or two elements, emit direct
340 // comparisons.
341 if (SecondFalseElement != Overdefined) {
342 Idx = MaskIdx(Idx);
343 // None false -> true.
344 if (FirstFalseElement == Undefined)
345 return replaceInstUsesWith(ICI, Builder.getTrue());
346
347 Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement);
348
349 // False for one element -> 'i != 47'.
350 if (SecondFalseElement == Undefined)
351 return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx);
352
353 // False for two elements -> 'i != 47 & i != 72'.
354 Value *C1 = Builder.CreateICmpNE(Idx, FirstFalseIdx);
355 Value *SecondFalseIdx =
356 ConstantInt::get(Idx->getType(), SecondFalseElement);
357 Value *C2 = Builder.CreateICmpNE(Idx, SecondFalseIdx);
358 return BinaryOperator::CreateAnd(C1, C2);
359 }
360
361 // If the comparison can be replaced with a range comparison for the elements
362 // where it is true, emit the range check.
363 if (TrueRangeEnd != Overdefined) {
364 assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare");
365 Idx = MaskIdx(Idx);
366
367 // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1).
368 if (FirstTrueElement) {
369 Value *Offs = ConstantInt::get(Idx->getType(), -FirstTrueElement);
370 Idx = Builder.CreateAdd(Idx, Offs);
371 }
372
373 Value *End =
374 ConstantInt::get(Idx->getType(), TrueRangeEnd - FirstTrueElement + 1);
375 return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End);
376 }
377
378 // False range check.
379 if (FalseRangeEnd != Overdefined) {
380 assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare");
381 Idx = MaskIdx(Idx);
382 // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse).
383 if (FirstFalseElement) {
384 Value *Offs = ConstantInt::get(Idx->getType(), -FirstFalseElement);
385 Idx = Builder.CreateAdd(Idx, Offs);
386 }
387
388 Value *End =
389 ConstantInt::get(Idx->getType(), FalseRangeEnd - FirstFalseElement);
390 return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End);
391 }
392
393 // If a magic bitvector captures the entire comparison state
394 // of this load, replace it with computation that does:
395 // ((magic_cst >> i) & 1) != 0
396 {
397 Type *Ty = nullptr;
398
399 // Look for an appropriate type:
400 // - The type of Idx if the magic fits
401 // - The smallest fitting legal type
402 if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth())
403 Ty = Idx->getType();
404 else
405 Ty = DL.getSmallestLegalIntType(Init->getContext(), ArrayElementCount);
406
407 if (Ty) {
408 Idx = MaskIdx(Idx);
409 Value *V = Builder.CreateIntCast(Idx, Ty, false);
410 V = Builder.CreateLShr(ConstantInt::get(Ty, MagicBitvector), V);
411 V = Builder.CreateAnd(ConstantInt::get(Ty, 1), V);
412 return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0));
413 }
414 }
415
416 return nullptr;
417}
418
419/// Returns true if we can rewrite Start as a GEP with pointer Base
420/// and some integer offset. The nodes that need to be re-written
421/// for this transformation will be added to Explored.
423 const DataLayout &DL,
424 SetVector<Value *> &Explored) {
425 SmallVector<Value *, 16> WorkList(1, Start);
426 Explored.insert(Base);
427
428 // The following traversal gives us an order which can be used
429 // when doing the final transformation. Since in the final
430 // transformation we create the PHI replacement instructions first,
431 // we don't have to get them in any particular order.
432 //
433 // However, for other instructions we will have to traverse the
434 // operands of an instruction first, which means that we have to
435 // do a post-order traversal.
436 while (!WorkList.empty()) {
438
439 while (!WorkList.empty()) {
440 if (Explored.size() >= 100)
441 return false;
442
443 Value *V = WorkList.back();
444
445 if (Explored.contains(V)) {
446 WorkList.pop_back();
447 continue;
448 }
449
450 if (!isa<GetElementPtrInst>(V) && !isa<PHINode>(V))
451 // We've found some value that we can't explore which is different from
452 // the base. Therefore we can't do this transformation.
453 return false;
454
455 if (auto *GEP = dyn_cast<GEPOperator>(V)) {
456 // Only allow inbounds GEPs with at most one variable offset.
457 auto IsNonConst = [](Value *V) { return !isa<ConstantInt>(V); };
458 if (!GEP->isInBounds() || count_if(GEP->indices(), IsNonConst) > 1)
459 return false;
460
461 if (!Explored.contains(GEP->getOperand(0)))
462 WorkList.push_back(GEP->getOperand(0));
463 }
464
465 if (WorkList.back() == V) {
466 WorkList.pop_back();
467 // We've finished visiting this node, mark it as such.
468 Explored.insert(V);
469 }
470
471 if (auto *PN = dyn_cast<PHINode>(V)) {
472 // We cannot transform PHIs on unsplittable basic blocks.
473 if (isa<CatchSwitchInst>(PN->getParent()->getTerminator()))
474 return false;
475 Explored.insert(PN);
476 PHIs.insert(PN);
477 }
478 }
479
480 // Explore the PHI nodes further.
481 for (auto *PN : PHIs)
482 for (Value *Op : PN->incoming_values())
483 if (!Explored.contains(Op))
484 WorkList.push_back(Op);
485 }
486
487 // Make sure that we can do this. Since we can't insert GEPs in a basic
488 // block before a PHI node, we can't easily do this transformation if
489 // we have PHI node users of transformed instructions.
490 for (Value *Val : Explored) {
491 for (Value *Use : Val->uses()) {
492
493 auto *PHI = dyn_cast<PHINode>(Use);
494 auto *Inst = dyn_cast<Instruction>(Val);
495
496 if (Inst == Base || Inst == PHI || !Inst || !PHI ||
497 !Explored.contains(PHI))
498 continue;
499
500 if (PHI->getParent() == Inst->getParent())
501 return false;
502 }
503 }
504 return true;
505}
506
507// Sets the appropriate insert point on Builder where we can add
508// a replacement Instruction for V (if that is possible).
509static void setInsertionPoint(IRBuilder<> &Builder, Value *V,
510 bool Before = true) {
511 if (auto *PHI = dyn_cast<PHINode>(V)) {
512 BasicBlock *Parent = PHI->getParent();
513 Builder.SetInsertPoint(Parent, Parent->getFirstInsertionPt());
514 return;
515 }
516 if (auto *I = dyn_cast<Instruction>(V)) {
517 if (!Before)
518 I = &*std::next(I->getIterator());
519 Builder.SetInsertPoint(I);
520 return;
521 }
522 if (auto *A = dyn_cast<Argument>(V)) {
523 // Set the insertion point in the entry block.
524 BasicBlock &Entry = A->getParent()->getEntryBlock();
525 Builder.SetInsertPoint(&Entry, Entry.getFirstInsertionPt());
526 return;
527 }
528 // Otherwise, this is a constant and we don't need to set a new
529 // insertion point.
530 assert(isa<Constant>(V) && "Setting insertion point for unknown value!");
531}
532
533/// Returns a re-written value of Start as an indexed GEP using Base as a
534/// pointer.
536 const DataLayout &DL,
537 SetVector<Value *> &Explored,
538 InstCombiner &IC) {
539 // Perform all the substitutions. This is a bit tricky because we can
540 // have cycles in our use-def chains.
541 // 1. Create the PHI nodes without any incoming values.
542 // 2. Create all the other values.
543 // 3. Add the edges for the PHI nodes.
544 // 4. Emit GEPs to get the original pointers.
545 // 5. Remove the original instructions.
546 Type *IndexType = IntegerType::get(
547 Base->getContext(), DL.getIndexTypeSizeInBits(Start->getType()));
548
550 NewInsts[Base] = ConstantInt::getNullValue(IndexType);
551
552 // Create the new PHI nodes, without adding any incoming values.
553 for (Value *Val : Explored) {
554 if (Val == Base)
555 continue;
556 // Create empty phi nodes. This avoids cyclic dependencies when creating
557 // the remaining instructions.
558 if (auto *PHI = dyn_cast<PHINode>(Val))
559 NewInsts[PHI] =
560 PHINode::Create(IndexType, PHI->getNumIncomingValues(),
561 PHI->getName() + ".idx", PHI->getIterator());
562 }
563 IRBuilder<> Builder(Base->getContext());
564
565 // Create all the other instructions.
566 for (Value *Val : Explored) {
567 if (NewInsts.contains(Val))
568 continue;
569
570 if (auto *GEP = dyn_cast<GEPOperator>(Val)) {
571 setInsertionPoint(Builder, GEP);
572 Value *Op = NewInsts[GEP->getOperand(0)];
573 Value *OffsetV = emitGEPOffset(&Builder, DL, GEP);
574 if (isa<ConstantInt>(Op) && cast<ConstantInt>(Op)->isZero())
575 NewInsts[GEP] = OffsetV;
576 else
577 NewInsts[GEP] = Builder.CreateNSWAdd(
578 Op, OffsetV, GEP->getOperand(0)->getName() + ".add");
579 continue;
580 }
581 if (isa<PHINode>(Val))
582 continue;
583
584 llvm_unreachable("Unexpected instruction type");
585 }
586
587 // Add the incoming values to the PHI nodes.
588 for (Value *Val : Explored) {
589 if (Val == Base)
590 continue;
591 // All the instructions have been created, we can now add edges to the
592 // phi nodes.
593 if (auto *PHI = dyn_cast<PHINode>(Val)) {
594 PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]);
595 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) {
596 Value *NewIncoming = PHI->getIncomingValue(I);
597
598 if (NewInsts.contains(NewIncoming))
599 NewIncoming = NewInsts[NewIncoming];
600
601 NewPhi->addIncoming(NewIncoming, PHI->getIncomingBlock(I));
602 }
603 }
604 }
605
606 for (Value *Val : Explored) {
607 if (Val == Base)
608 continue;
609
610 setInsertionPoint(Builder, Val, false);
611 // Create GEP for external users.
612 Value *NewVal = Builder.CreateInBoundsGEP(
613 Builder.getInt8Ty(), Base, NewInsts[Val], Val->getName() + ".ptr");
614 IC.replaceInstUsesWith(*cast<Instruction>(Val), NewVal);
615 // Add old instruction to worklist for DCE. We don't directly remove it
616 // here because the original compare is one of the users.
617 IC.addToWorklist(cast<Instruction>(Val));
618 }
619
620 return NewInsts[Start];
621}
622
623/// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant.
624/// We can look through PHIs, GEPs and casts in order to determine a common base
625/// between GEPLHS and RHS.
628 const DataLayout &DL,
629 InstCombiner &IC) {
630 // FIXME: Support vector of pointers.
631 if (GEPLHS->getType()->isVectorTy())
632 return nullptr;
633
634 if (!GEPLHS->hasAllConstantIndices())
635 return nullptr;
636
637 APInt Offset(DL.getIndexTypeSizeInBits(GEPLHS->getType()), 0);
638 Value *PtrBase =
640 /*AllowNonInbounds*/ false);
641
642 // Bail if we looked through addrspacecast.
643 if (PtrBase->getType() != GEPLHS->getType())
644 return nullptr;
645
646 // The set of nodes that will take part in this transformation.
647 SetVector<Value *> Nodes;
648
649 if (!canRewriteGEPAsOffset(RHS, PtrBase, DL, Nodes))
650 return nullptr;
651
652 // We know we can re-write this as
653 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2)
654 // Since we've only looked through inbouds GEPs we know that we
655 // can't have overflow on either side. We can therefore re-write
656 // this as:
657 // OFFSET1 cmp OFFSET2
658 Value *NewRHS = rewriteGEPAsOffset(RHS, PtrBase, DL, Nodes, IC);
659
660 // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written
661 // GEP having PtrBase as the pointer base, and has returned in NewRHS the
662 // offset. Since Index is the offset of LHS to the base pointer, we will now
663 // compare the offsets instead of comparing the pointers.
665 IC.Builder.getInt(Offset), NewRHS);
666}
667
668/// Fold comparisons between a GEP instruction and something else. At this point
669/// we know that the GEP is on the LHS of the comparison.
672 Instruction &I) {
673 // Don't transform signed compares of GEPs into index compares. Even if the
674 // GEP is inbounds, the final add of the base pointer can have signed overflow
675 // and would change the result of the icmp.
676 // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be
677 // the maximum signed value for the pointer type.
679 return nullptr;
680
681 // Look through bitcasts and addrspacecasts. We do not however want to remove
682 // 0 GEPs.
683 if (!isa<GetElementPtrInst>(RHS))
685
686 Value *PtrBase = GEPLHS->getOperand(0);
687 if (PtrBase == RHS && (GEPLHS->isInBounds() || ICmpInst::isEquality(Cond))) {
688 // ((gep Ptr, OFFSET) cmp Ptr) ---> (OFFSET cmp 0).
689 Value *Offset = EmitGEPOffset(GEPLHS);
691 Constant::getNullValue(Offset->getType()));
692 }
693
694 if (GEPLHS->isInBounds() && ICmpInst::isEquality(Cond) &&
695 isa<Constant>(RHS) && cast<Constant>(RHS)->isNullValue() &&
696 !NullPointerIsDefined(I.getFunction(),
698 // For most address spaces, an allocation can't be placed at null, but null
699 // itself is treated as a 0 size allocation in the in bounds rules. Thus,
700 // the only valid inbounds address derived from null, is null itself.
701 // Thus, we have four cases to consider:
702 // 1) Base == nullptr, Offset == 0 -> inbounds, null
703 // 2) Base == nullptr, Offset != 0 -> poison as the result is out of bounds
704 // 3) Base != nullptr, Offset == (-base) -> poison (crossing allocations)
705 // 4) Base != nullptr, Offset != (-base) -> nonnull (and possibly poison)
706 //
707 // (Note if we're indexing a type of size 0, that simply collapses into one
708 // of the buckets above.)
709 //
710 // In general, we're allowed to make values less poison (i.e. remove
711 // sources of full UB), so in this case, we just select between the two
712 // non-poison cases (1 and 4 above).
713 //
714 // For vectors, we apply the same reasoning on a per-lane basis.
715 auto *Base = GEPLHS->getPointerOperand();
716 if (GEPLHS->getType()->isVectorTy() && Base->getType()->isPointerTy()) {
717 auto EC = cast<VectorType>(GEPLHS->getType())->getElementCount();
719 }
720 return new ICmpInst(Cond, Base,
722 cast<Constant>(RHS), Base->getType()));
723 } else if (GEPOperator *GEPRHS = dyn_cast<GEPOperator>(RHS)) {
724 // If the base pointers are different, but the indices are the same, just
725 // compare the base pointer.
726 if (PtrBase != GEPRHS->getOperand(0)) {
727 bool IndicesTheSame =
728 GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
729 GEPLHS->getPointerOperand()->getType() ==
730 GEPRHS->getPointerOperand()->getType() &&
731 GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType();
732 if (IndicesTheSame)
733 for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i)
734 if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) {
735 IndicesTheSame = false;
736 break;
737 }
738
739 // If all indices are the same, just compare the base pointers.
740 Type *BaseType = GEPLHS->getOperand(0)->getType();
741 if (IndicesTheSame && CmpInst::makeCmpResultType(BaseType) == I.getType())
742 return new ICmpInst(Cond, GEPLHS->getOperand(0), GEPRHS->getOperand(0));
743
744 // If we're comparing GEPs with two base pointers that only differ in type
745 // and both GEPs have only constant indices or just one use, then fold
746 // the compare with the adjusted indices.
747 // FIXME: Support vector of pointers.
748 if (GEPLHS->isInBounds() && GEPRHS->isInBounds() &&
749 (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) &&
750 (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) &&
751 PtrBase->stripPointerCasts() ==
752 GEPRHS->getOperand(0)->stripPointerCasts() &&
753 !GEPLHS->getType()->isVectorTy()) {
754 Value *LOffset = EmitGEPOffset(GEPLHS);
755 Value *ROffset = EmitGEPOffset(GEPRHS);
756
757 // If we looked through an addrspacecast between different sized address
758 // spaces, the LHS and RHS pointers are different sized
759 // integers. Truncate to the smaller one.
760 Type *LHSIndexTy = LOffset->getType();
761 Type *RHSIndexTy = ROffset->getType();
762 if (LHSIndexTy != RHSIndexTy) {
763 if (LHSIndexTy->getPrimitiveSizeInBits().getFixedValue() <
764 RHSIndexTy->getPrimitiveSizeInBits().getFixedValue()) {
765 ROffset = Builder.CreateTrunc(ROffset, LHSIndexTy);
766 } else
767 LOffset = Builder.CreateTrunc(LOffset, RHSIndexTy);
768 }
769
771 LOffset, ROffset);
772 return replaceInstUsesWith(I, Cmp);
773 }
774
775 // Otherwise, the base pointers are different and the indices are
776 // different. Try convert this to an indexed compare by looking through
777 // PHIs/casts.
778 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL, *this);
779 }
780
781 bool GEPsInBounds = GEPLHS->isInBounds() && GEPRHS->isInBounds();
782 if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
783 GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType()) {
784 // If the GEPs only differ by one index, compare it.
785 unsigned NumDifferences = 0; // Keep track of # differences.
786 unsigned DiffOperand = 0; // The operand that differs.
787 for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i)
788 if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) {
789 Type *LHSType = GEPLHS->getOperand(i)->getType();
790 Type *RHSType = GEPRHS->getOperand(i)->getType();
791 // FIXME: Better support for vector of pointers.
792 if (LHSType->getPrimitiveSizeInBits() !=
793 RHSType->getPrimitiveSizeInBits() ||
794 (GEPLHS->getType()->isVectorTy() &&
795 (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) {
796 // Irreconcilable differences.
797 NumDifferences = 2;
798 break;
799 }
800
801 if (NumDifferences++) break;
802 DiffOperand = i;
803 }
804
805 if (NumDifferences == 0) // SAME GEP?
806 return replaceInstUsesWith(I, // No comparison is needed here.
807 ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond)));
808
809 else if (NumDifferences == 1 && GEPsInBounds) {
810 Value *LHSV = GEPLHS->getOperand(DiffOperand);
811 Value *RHSV = GEPRHS->getOperand(DiffOperand);
812 // Make sure we do a signed comparison here.
813 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), LHSV, RHSV);
814 }
815 }
816
817 if (GEPsInBounds || CmpInst::isEquality(Cond)) {
818 auto EmitGEPOffsetAndRewrite = [&](GEPOperator *GEP) {
820 auto *Inst = dyn_cast<Instruction>(GEP);
821 if (Inst)
823
824 Value *Offset = EmitGEPOffset(GEP);
825 // If a non-trivial GEP has other uses, rewrite it to avoid duplicating
826 // the offset arithmetic.
827 if (Inst && !GEP->hasOneUse() && !GEP->hasAllConstantIndices() &&
828 !GEP->getSourceElementType()->isIntegerTy(8)) {
831 GEP->getPointerOperand(),
832 Offset, "", GEPsInBounds));
834 }
835 return Offset;
836 };
837
838 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2)
839 Value *L = EmitGEPOffsetAndRewrite(GEPLHS);
840 Value *R = EmitGEPOffsetAndRewrite(GEPRHS);
841 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), L, R);
842 }
843 }
844
845 // Try convert this to an indexed compare by looking through PHIs/casts as a
846 // last resort.
847 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL, *this);
848}
849
851 // It would be tempting to fold away comparisons between allocas and any
852 // pointer not based on that alloca (e.g. an argument). However, even
853 // though such pointers cannot alias, they can still compare equal.
854 //
855 // But LLVM doesn't specify where allocas get their memory, so if the alloca
856 // doesn't escape we can argue that it's impossible to guess its value, and we
857 // can therefore act as if any such guesses are wrong.
858 //
859 // However, we need to ensure that this folding is consistent: We can't fold
860 // one comparison to false, and then leave a different comparison against the
861 // same value alone (as it might evaluate to true at runtime, leading to a
862 // contradiction). As such, this code ensures that all comparisons are folded
863 // at the same time, and there are no other escapes.
864
865 struct CmpCaptureTracker : public CaptureTracker {
866 AllocaInst *Alloca;
867 bool Captured = false;
868 /// The value of the map is a bit mask of which icmp operands the alloca is
869 /// used in.
871
872 CmpCaptureTracker(AllocaInst *Alloca) : Alloca(Alloca) {}
873
874 void tooManyUses() override { Captured = true; }
875
876 bool captured(const Use *U) override {
877 auto *ICmp = dyn_cast<ICmpInst>(U->getUser());
878 // We need to check that U is based *only* on the alloca, and doesn't
879 // have other contributions from a select/phi operand.
880 // TODO: We could check whether getUnderlyingObjects() reduces to one
881 // object, which would allow looking through phi nodes.
882 if (ICmp && ICmp->isEquality() && getUnderlyingObject(*U) == Alloca) {
883 // Collect equality icmps of the alloca, and don't treat them as
884 // captures.
885 auto Res = ICmps.insert({ICmp, 0});
886 Res.first->second |= 1u << U->getOperandNo();
887 return false;
888 }
889
890 Captured = true;
891 return true;
892 }
893 };
894
895 CmpCaptureTracker Tracker(Alloca);
896 PointerMayBeCaptured(Alloca, &Tracker);
897 if (Tracker.Captured)
898 return false;
899
900 bool Changed = false;
901 for (auto [ICmp, Operands] : Tracker.ICmps) {
902 switch (Operands) {
903 case 1:
904 case 2: {
905 // The alloca is only used in one icmp operand. Assume that the
906 // equality is false.
907 auto *Res = ConstantInt::get(
908 ICmp->getType(), ICmp->getPredicate() == ICmpInst::ICMP_NE);
909 replaceInstUsesWith(*ICmp, Res);
911 Changed = true;
912 break;
913 }
914 case 3:
915 // Both icmp operands are based on the alloca, so this is comparing
916 // pointer offsets, without leaking any information about the address
917 // of the alloca. Ignore such comparisons.
918 break;
919 default:
920 llvm_unreachable("Cannot happen");
921 }
922 }
923
924 return Changed;
925}
926
927/// Fold "icmp pred (X+C), X".
929 ICmpInst::Predicate Pred) {
930 // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0,
931 // so the values can never be equal. Similarly for all other "or equals"
932 // operators.
933 assert(!!C && "C should not be zero!");
934
935 // (X+1) <u X --> X >u (MAXUINT-1) --> X == 255
936 // (X+2) <u X --> X >u (MAXUINT-2) --> X > 253
937 // (X+MAXUINT) <u X --> X >u (MAXUINT-MAXUINT) --> X != 0
938 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
939 Constant *R = ConstantInt::get(X->getType(),
940 APInt::getMaxValue(C.getBitWidth()) - C);
941 return new ICmpInst(ICmpInst::ICMP_UGT, X, R);
942 }
943
944 // (X+1) >u X --> X <u (0-1) --> X != 255
945 // (X+2) >u X --> X <u (0-2) --> X <u 254
946 // (X+MAXUINT) >u X --> X <u (0-MAXUINT) --> X <u 1 --> X == 0
947 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
948 return new ICmpInst(ICmpInst::ICMP_ULT, X,
949 ConstantInt::get(X->getType(), -C));
950
951 APInt SMax = APInt::getSignedMaxValue(C.getBitWidth());
952
953 // (X+ 1) <s X --> X >s (MAXSINT-1) --> X == 127
954 // (X+ 2) <s X --> X >s (MAXSINT-2) --> X >s 125
955 // (X+MAXSINT) <s X --> X >s (MAXSINT-MAXSINT) --> X >s 0
956 // (X+MINSINT) <s X --> X >s (MAXSINT-MINSINT) --> X >s -1
957 // (X+ -2) <s X --> X >s (MAXSINT- -2) --> X >s 126
958 // (X+ -1) <s X --> X >s (MAXSINT- -1) --> X != 127
959 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
960 return new ICmpInst(ICmpInst::ICMP_SGT, X,
961 ConstantInt::get(X->getType(), SMax - C));
962
963 // (X+ 1) >s X --> X <s (MAXSINT-(1-1)) --> X != 127
964 // (X+ 2) >s X --> X <s (MAXSINT-(2-1)) --> X <s 126
965 // (X+MAXSINT) >s X --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1
966 // (X+MINSINT) >s X --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2
967 // (X+ -2) >s X --> X <s (MAXSINT-(-2-1)) --> X <s -126
968 // (X+ -1) >s X --> X <s (MAXSINT-(-1-1)) --> X == -128
969
970 assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE);
971 return new ICmpInst(ICmpInst::ICMP_SLT, X,
972 ConstantInt::get(X->getType(), SMax - (C - 1)));
973}
974
975/// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" ->
976/// (icmp eq/ne A, Log2(AP2/AP1)) ->
977/// (icmp eq/ne A, Log2(AP2) - Log2(AP1)).
979 const APInt &AP1,
980 const APInt &AP2) {
981 assert(I.isEquality() && "Cannot fold icmp gt/lt");
982
983 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
984 if (I.getPredicate() == I.ICMP_NE)
985 Pred = CmpInst::getInversePredicate(Pred);
986 return new ICmpInst(Pred, LHS, RHS);
987 };
988
989 // Don't bother doing any work for cases which InstSimplify handles.
990 if (AP2.isZero())
991 return nullptr;
992
993 bool IsAShr = isa<AShrOperator>(I.getOperand(0));
994 if (IsAShr) {
995 if (AP2.isAllOnes())
996 return nullptr;
997 if (AP2.isNegative() != AP1.isNegative())
998 return nullptr;
999 if (AP2.sgt(AP1))
1000 return nullptr;
1001 }
1002
1003 if (!AP1)
1004 // 'A' must be large enough to shift out the highest set bit.
1005 return getICmp(I.ICMP_UGT, A,
1006 ConstantInt::get(A->getType(), AP2.logBase2()));
1007
1008 if (AP1 == AP2)
1009 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType()));
1010
1011 int Shift;
1012 if (IsAShr && AP1.isNegative())
1013 Shift = AP1.countl_one() - AP2.countl_one();
1014 else
1015 Shift = AP1.countl_zero() - AP2.countl_zero();
1016
1017 if (Shift > 0) {
1018 if (IsAShr && AP1 == AP2.ashr(Shift)) {
1019 // There are multiple solutions if we are comparing against -1 and the LHS
1020 // of the ashr is not a power of two.
1021 if (AP1.isAllOnes() && !AP2.isPowerOf2())
1022 return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift));
1023 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1024 } else if (AP1 == AP2.lshr(Shift)) {
1025 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1026 }
1027 }
1028
1029 // Shifting const2 will never be equal to const1.
1030 // FIXME: This should always be handled by InstSimplify?
1031 auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE);
1032 return replaceInstUsesWith(I, TorF);
1033}
1034
1035/// Handle "(icmp eq/ne (shl AP2, A), AP1)" ->
1036/// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)).
1038 const APInt &AP1,
1039 const APInt &AP2) {
1040 assert(I.isEquality() && "Cannot fold icmp gt/lt");
1041
1042 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
1043 if (I.getPredicate() == I.ICMP_NE)
1044 Pred = CmpInst::getInversePredicate(Pred);
1045 return new ICmpInst(Pred, LHS, RHS);
1046 };
1047
1048 // Don't bother doing any work for cases which InstSimplify handles.
1049 if (AP2.isZero())
1050 return nullptr;
1051
1052 unsigned AP2TrailingZeros = AP2.countr_zero();
1053
1054 if (!AP1 && AP2TrailingZeros != 0)
1055 return getICmp(
1056 I.ICMP_UGE, A,
1057 ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros));
1058
1059 if (AP1 == AP2)
1060 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType()));
1061
1062 // Get the distance between the lowest bits that are set.
1063 int Shift = AP1.countr_zero() - AP2TrailingZeros;
1064
1065 if (Shift > 0 && AP2.shl(Shift) == AP1)
1066 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1067
1068 // Shifting const2 will never be equal to const1.
1069 // FIXME: This should always be handled by InstSimplify?
1070 auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE);
1071 return replaceInstUsesWith(I, TorF);
1072}
1073
1074/// The caller has matched a pattern of the form:
1075/// I = icmp ugt (add (add A, B), CI2), CI1
1076/// If this is of the form:
1077/// sum = a + b
1078/// if (sum+128 >u 255)
1079/// Then replace it with llvm.sadd.with.overflow.i8.
1080///
1082 ConstantInt *CI2, ConstantInt *CI1,
1083 InstCombinerImpl &IC) {
1084 // The transformation we're trying to do here is to transform this into an
1085 // llvm.sadd.with.overflow. To do this, we have to replace the original add
1086 // with a narrower add, and discard the add-with-constant that is part of the
1087 // range check (if we can't eliminate it, this isn't profitable).
1088
1089 // In order to eliminate the add-with-constant, the compare can be its only
1090 // use.
1091 Instruction *AddWithCst = cast<Instruction>(I.getOperand(0));
1092 if (!AddWithCst->hasOneUse())
1093 return nullptr;
1094
1095 // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow.
1096 if (!CI2->getValue().isPowerOf2())
1097 return nullptr;
1098 unsigned NewWidth = CI2->getValue().countr_zero();
1099 if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31)
1100 return nullptr;
1101
1102 // The width of the new add formed is 1 more than the bias.
1103 ++NewWidth;
1104
1105 // Check to see that CI1 is an all-ones value with NewWidth bits.
1106 if (CI1->getBitWidth() == NewWidth ||
1107 CI1->getValue() != APInt::getLowBitsSet(CI1->getBitWidth(), NewWidth))
1108 return nullptr;
1109
1110 // This is only really a signed overflow check if the inputs have been
1111 // sign-extended; check for that condition. For example, if CI2 is 2^31 and
1112 // the operands of the add are 64 bits wide, we need at least 33 sign bits.
1113 if (IC.ComputeMaxSignificantBits(A, 0, &I) > NewWidth ||
1114 IC.ComputeMaxSignificantBits(B, 0, &I) > NewWidth)
1115 return nullptr;
1116
1117 // In order to replace the original add with a narrower
1118 // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant
1119 // and truncates that discard the high bits of the add. Verify that this is
1120 // the case.
1121 Instruction *OrigAdd = cast<Instruction>(AddWithCst->getOperand(0));
1122 for (User *U : OrigAdd->users()) {
1123 if (U == AddWithCst)
1124 continue;
1125
1126 // Only accept truncates for now. We would really like a nice recursive
1127 // predicate like SimplifyDemandedBits, but which goes downwards the use-def
1128 // chain to see which bits of a value are actually demanded. If the
1129 // original add had another add which was then immediately truncated, we
1130 // could still do the transformation.
1131 TruncInst *TI = dyn_cast<TruncInst>(U);
1132 if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth)
1133 return nullptr;
1134 }
1135
1136 // If the pattern matches, truncate the inputs to the narrower type and
1137 // use the sadd_with_overflow intrinsic to efficiently compute both the
1138 // result and the overflow bit.
1139 Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth);
1141 I.getModule(), Intrinsic::sadd_with_overflow, NewType);
1142
1143 InstCombiner::BuilderTy &Builder = IC.Builder;
1144
1145 // Put the new code above the original add, in case there are any uses of the
1146 // add between the add and the compare.
1147 Builder.SetInsertPoint(OrigAdd);
1148
1149 Value *TruncA = Builder.CreateTrunc(A, NewType, A->getName() + ".trunc");
1150 Value *TruncB = Builder.CreateTrunc(B, NewType, B->getName() + ".trunc");
1151 CallInst *Call = Builder.CreateCall(F, {TruncA, TruncB}, "sadd");
1152 Value *Add = Builder.CreateExtractValue(Call, 0, "sadd.result");
1153 Value *ZExt = Builder.CreateZExt(Add, OrigAdd->getType());
1154
1155 // The inner add was the result of the narrow add, zero extended to the
1156 // wider type. Replace it with the result computed by the intrinsic.
1157 IC.replaceInstUsesWith(*OrigAdd, ZExt);
1158 IC.eraseInstFromFunction(*OrigAdd);
1159
1160 // The original icmp gets replaced with the overflow value.
1161 return ExtractValueInst::Create(Call, 1, "sadd.overflow");
1162}
1163
1164/// If we have:
1165/// icmp eq/ne (urem/srem %x, %y), 0
1166/// iff %y is a power-of-two, we can replace this with a bit test:
1167/// icmp eq/ne (and %x, (add %y, -1)), 0
1169 // This fold is only valid for equality predicates.
1170 if (!I.isEquality())
1171 return nullptr;
1173 Value *X, *Y, *Zero;
1174 if (!match(&I, m_ICmp(Pred, m_OneUse(m_IRem(m_Value(X), m_Value(Y))),
1175 m_CombineAnd(m_Zero(), m_Value(Zero)))))
1176 return nullptr;
1177 if (!isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, 0, &I))
1178 return nullptr;
1179 // This may increase instruction count, we don't enforce that Y is a constant.
1180 Value *Mask = Builder.CreateAdd(Y, Constant::getAllOnesValue(Y->getType()));
1181 Value *Masked = Builder.CreateAnd(X, Mask);
1182 return ICmpInst::Create(Instruction::ICmp, Pred, Masked, Zero);
1183}
1184
1185/// Fold equality-comparison between zero and any (maybe truncated) right-shift
1186/// by one-less-than-bitwidth into a sign test on the original value.
1188 Instruction *Val;
1190 if (!I.isEquality() || !match(&I, m_ICmp(Pred, m_Instruction(Val), m_Zero())))
1191 return nullptr;
1192
1193 Value *X;
1194 Type *XTy;
1195
1196 Constant *C;
1197 if (match(Val, m_TruncOrSelf(m_Shr(m_Value(X), m_Constant(C))))) {
1198 XTy = X->getType();
1199 unsigned XBitWidth = XTy->getScalarSizeInBits();
1201 APInt(XBitWidth, XBitWidth - 1))))
1202 return nullptr;
1203 } else if (isa<BinaryOperator>(Val) &&
1205 cast<BinaryOperator>(Val), SQ.getWithInstruction(Val),
1206 /*AnalyzeForSignBitExtraction=*/true))) {
1207 XTy = X->getType();
1208 } else
1209 return nullptr;
1210
1211 return ICmpInst::Create(Instruction::ICmp,
1215}
1216
1217// Handle icmp pred X, 0
1219 CmpInst::Predicate Pred = Cmp.getPredicate();
1220 if (!match(Cmp.getOperand(1), m_Zero()))
1221 return nullptr;
1222
1223 // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0)
1224 if (Pred == ICmpInst::ICMP_SGT) {
1225 Value *A, *B;
1226 if (match(Cmp.getOperand(0), m_SMin(m_Value(A), m_Value(B)))) {
1228 return new ICmpInst(Pred, B, Cmp.getOperand(1));
1230 return new ICmpInst(Pred, A, Cmp.getOperand(1));
1231 }
1232 }
1233
1235 return New;
1236
1237 // Given:
1238 // icmp eq/ne (urem %x, %y), 0
1239 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
1240 // icmp eq/ne %x, 0
1241 Value *X, *Y;
1242 if (match(Cmp.getOperand(0), m_URem(m_Value(X), m_Value(Y))) &&
1243 ICmpInst::isEquality(Pred)) {
1244 KnownBits XKnown = computeKnownBits(X, 0, &Cmp);
1245 KnownBits YKnown = computeKnownBits(Y, 0, &Cmp);
1246 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
1247 return new ICmpInst(Pred, X, Cmp.getOperand(1));
1248 }
1249
1250 // (icmp eq/ne (mul X Y)) -> (icmp eq/ne X/Y) if we know about whether X/Y are
1251 // odd/non-zero/there is no overflow.
1252 if (match(Cmp.getOperand(0), m_Mul(m_Value(X), m_Value(Y))) &&
1253 ICmpInst::isEquality(Pred)) {
1254
1255 KnownBits XKnown = computeKnownBits(X, 0, &Cmp);
1256 // if X % 2 != 0
1257 // (icmp eq/ne Y)
1258 if (XKnown.countMaxTrailingZeros() == 0)
1259 return new ICmpInst(Pred, Y, Cmp.getOperand(1));
1260
1261 KnownBits YKnown = computeKnownBits(Y, 0, &Cmp);
1262 // if Y % 2 != 0
1263 // (icmp eq/ne X)
1264 if (YKnown.countMaxTrailingZeros() == 0)
1265 return new ICmpInst(Pred, X, Cmp.getOperand(1));
1266
1267 auto *BO0 = cast<OverflowingBinaryOperator>(Cmp.getOperand(0));
1268 if (BO0->hasNoUnsignedWrap() || BO0->hasNoSignedWrap()) {
1269 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
1270 // `isKnownNonZero` does more analysis than just `!KnownBits.One.isZero()`
1271 // but to avoid unnecessary work, first just if this is an obvious case.
1272
1273 // if X non-zero and NoOverflow(X * Y)
1274 // (icmp eq/ne Y)
1275 if (!XKnown.One.isZero() || isKnownNonZero(X, DL, 0, Q.AC, Q.CxtI, Q.DT))
1276 return new ICmpInst(Pred, Y, Cmp.getOperand(1));
1277
1278 // if Y non-zero and NoOverflow(X * Y)
1279 // (icmp eq/ne X)
1280 if (!YKnown.One.isZero() || isKnownNonZero(Y, DL, 0, Q.AC, Q.CxtI, Q.DT))
1281 return new ICmpInst(Pred, X, Cmp.getOperand(1));
1282 }
1283 // Note, we are skipping cases:
1284 // if Y % 2 != 0 AND X % 2 != 0
1285 // (false/true)
1286 // if X non-zero and Y non-zero and NoOverflow(X * Y)
1287 // (false/true)
1288 // Those can be simplified later as we would have already replaced the (icmp
1289 // eq/ne (mul X, Y)) with (icmp eq/ne X/Y) and if X/Y is known non-zero that
1290 // will fold to a constant elsewhere.
1291 }
1292 return nullptr;
1293}
1294
1295/// Fold icmp Pred X, C.
1296/// TODO: This code structure does not make sense. The saturating add fold
1297/// should be moved to some other helper and extended as noted below (it is also
1298/// possible that code has been made unnecessary - do we canonicalize IR to
1299/// overflow/saturating intrinsics or not?).
1301 // Match the following pattern, which is a common idiom when writing
1302 // overflow-safe integer arithmetic functions. The source performs an addition
1303 // in wider type and explicitly checks for overflow using comparisons against
1304 // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic.
1305 //
1306 // TODO: This could probably be generalized to handle other overflow-safe
1307 // operations if we worked out the formulas to compute the appropriate magic
1308 // constants.
1309 //
1310 // sum = a + b
1311 // if (sum+128 >u 255) ... -> llvm.sadd.with.overflow.i8
1312 CmpInst::Predicate Pred = Cmp.getPredicate();
1313 Value *Op0 = Cmp.getOperand(0), *Op1 = Cmp.getOperand(1);
1314 Value *A, *B;
1315 ConstantInt *CI, *CI2; // I = icmp ugt (add (add A, B), CI2), CI
1316 if (Pred == ICmpInst::ICMP_UGT && match(Op1, m_ConstantInt(CI)) &&
1317 match(Op0, m_Add(m_Add(m_Value(A), m_Value(B)), m_ConstantInt(CI2))))
1318 if (Instruction *Res = processUGT_ADDCST_ADD(Cmp, A, B, CI2, CI, *this))
1319 return Res;
1320
1321 // icmp(phi(C1, C2, ...), C) -> phi(icmp(C1, C), icmp(C2, C), ...).
1322 Constant *C = dyn_cast<Constant>(Op1);
1323 if (!C)
1324 return nullptr;
1325
1326 if (auto *Phi = dyn_cast<PHINode>(Op0))
1327 if (all_of(Phi->operands(), [](Value *V) { return isa<Constant>(V); })) {
1329 for (Value *V : Phi->incoming_values()) {
1330 Constant *Res =
1331 ConstantFoldCompareInstOperands(Pred, cast<Constant>(V), C, DL);
1332 if (!Res)
1333 return nullptr;
1334 Ops.push_back(Res);
1335 }
1337 PHINode *NewPhi = Builder.CreatePHI(Cmp.getType(), Phi->getNumOperands());
1338 for (auto [V, Pred] : zip(Ops, Phi->blocks()))
1339 NewPhi->addIncoming(V, Pred);
1340 return replaceInstUsesWith(Cmp, NewPhi);
1341 }
1342
1344 return R;
1345
1346 return nullptr;
1347}
1348
1349/// Canonicalize icmp instructions based on dominating conditions.
1351 // We already checked simple implication in InstSimplify, only handle complex
1352 // cases here.
1353 Value *X = Cmp.getOperand(0), *Y = Cmp.getOperand(1);
1354 const APInt *C;
1355 if (!match(Y, m_APInt(C)))
1356 return nullptr;
1357
1358 CmpInst::Predicate Pred = Cmp.getPredicate();
1360
1361 auto handleDomCond = [&](ICmpInst::Predicate DomPred,
1362 const APInt *DomC) -> Instruction * {
1363 // We have 2 compares of a variable with constants. Calculate the constant
1364 // ranges of those compares to see if we can transform the 2nd compare:
1365 // DomBB:
1366 // DomCond = icmp DomPred X, DomC
1367 // br DomCond, CmpBB, FalseBB
1368 // CmpBB:
1369 // Cmp = icmp Pred X, C
1370 ConstantRange DominatingCR =
1371 ConstantRange::makeExactICmpRegion(DomPred, *DomC);
1372 ConstantRange Intersection = DominatingCR.intersectWith(CR);
1373 ConstantRange Difference = DominatingCR.difference(CR);
1374 if (Intersection.isEmptySet())
1375 return replaceInstUsesWith(Cmp, Builder.getFalse());
1376 if (Difference.isEmptySet())
1377 return replaceInstUsesWith(Cmp, Builder.getTrue());
1378
1379 // Canonicalizing a sign bit comparison that gets used in a branch,
1380 // pessimizes codegen by generating branch on zero instruction instead
1381 // of a test and branch. So we avoid canonicalizing in such situations
1382 // because test and branch instruction has better branch displacement
1383 // than compare and branch instruction.
1384 bool UnusedBit;
1385 bool IsSignBit = isSignBitCheck(Pred, *C, UnusedBit);
1386 if (Cmp.isEquality() || (IsSignBit && hasBranchUse(Cmp)))
1387 return nullptr;
1388
1389 // Avoid an infinite loop with min/max canonicalization.
1390 // TODO: This will be unnecessary if we canonicalize to min/max intrinsics.
1391 if (Cmp.hasOneUse() &&
1392 match(Cmp.user_back(), m_MaxOrMin(m_Value(), m_Value())))
1393 return nullptr;
1394
1395 if (const APInt *EqC = Intersection.getSingleElement())
1396 return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(*EqC));
1397 if (const APInt *NeC = Difference.getSingleElement())
1398 return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(*NeC));
1399 return nullptr;
1400 };
1401
1402 for (BranchInst *BI : DC.conditionsFor(X)) {
1403 ICmpInst::Predicate DomPred;
1404 const APInt *DomC;
1405 if (!match(BI->getCondition(),
1406 m_ICmp(DomPred, m_Specific(X), m_APInt(DomC))))
1407 continue;
1408
1409 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0));
1410 if (DT.dominates(Edge0, Cmp.getParent())) {
1411 if (auto *V = handleDomCond(DomPred, DomC))
1412 return V;
1413 } else {
1414 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1));
1415 if (DT.dominates(Edge1, Cmp.getParent()))
1416 if (auto *V =
1417 handleDomCond(CmpInst::getInversePredicate(DomPred), DomC))
1418 return V;
1419 }
1420 }
1421
1422 return nullptr;
1423}
1424
1425/// Fold icmp (trunc X), C.
1427 TruncInst *Trunc,
1428 const APInt &C) {
1429 ICmpInst::Predicate Pred = Cmp.getPredicate();
1430 Value *X = Trunc->getOperand(0);
1431 if (C.isOne() && C.getBitWidth() > 1) {
1432 // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1
1433 Value *V = nullptr;
1434 if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V))))
1435 return new ICmpInst(ICmpInst::ICMP_SLT, V,
1436 ConstantInt::get(V->getType(), 1));
1437 }
1438
1439 Type *SrcTy = X->getType();
1440 unsigned DstBits = Trunc->getType()->getScalarSizeInBits(),
1441 SrcBits = SrcTy->getScalarSizeInBits();
1442
1443 // TODO: Handle any shifted constant by subtracting trailing zeros.
1444 // TODO: Handle non-equality predicates.
1445 Value *Y;
1446 if (Cmp.isEquality() && match(X, m_Shl(m_One(), m_Value(Y)))) {
1447 // (trunc (1 << Y) to iN) == 0 --> Y u>= N
1448 // (trunc (1 << Y) to iN) != 0 --> Y u< N
1449 if (C.isZero()) {
1450 auto NewPred = (Pred == Cmp.ICMP_EQ) ? Cmp.ICMP_UGE : Cmp.ICMP_ULT;
1451 return new ICmpInst(NewPred, Y, ConstantInt::get(SrcTy, DstBits));
1452 }
1453 // (trunc (1 << Y) to iN) == 2**C --> Y == C
1454 // (trunc (1 << Y) to iN) != 2**C --> Y != C
1455 if (C.isPowerOf2())
1456 return new ICmpInst(Pred, Y, ConstantInt::get(SrcTy, C.logBase2()));
1457 }
1458
1459 if (Cmp.isEquality() && Trunc->hasOneUse()) {
1460 // Canonicalize to a mask and wider compare if the wide type is suitable:
1461 // (trunc X to i8) == C --> (X & 0xff) == (zext C)
1462 if (!SrcTy->isVectorTy() && shouldChangeType(DstBits, SrcBits)) {
1463 Constant *Mask =
1464 ConstantInt::get(SrcTy, APInt::getLowBitsSet(SrcBits, DstBits));
1465 Value *And = Builder.CreateAnd(X, Mask);
1466 Constant *WideC = ConstantInt::get(SrcTy, C.zext(SrcBits));
1467 return new ICmpInst(Pred, And, WideC);
1468 }
1469
1470 // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all
1471 // of the high bits truncated out of x are known.
1472 KnownBits Known = computeKnownBits(X, 0, &Cmp);
1473
1474 // If all the high bits are known, we can do this xform.
1475 if ((Known.Zero | Known.One).countl_one() >= SrcBits - DstBits) {
1476 // Pull in the high bits from known-ones set.
1477 APInt NewRHS = C.zext(SrcBits);
1478 NewRHS |= Known.One & APInt::getHighBitsSet(SrcBits, SrcBits - DstBits);
1479 return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, NewRHS));
1480 }
1481 }
1482
1483 // Look through truncated right-shift of the sign-bit for a sign-bit check:
1484 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] < 0 --> ShOp < 0
1485 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] > -1 --> ShOp > -1
1486 Value *ShOp;
1487 const APInt *ShAmtC;
1488 bool TrueIfSigned;
1489 if (isSignBitCheck(Pred, C, TrueIfSigned) &&
1490 match(X, m_Shr(m_Value(ShOp), m_APInt(ShAmtC))) &&
1491 DstBits == SrcBits - ShAmtC->getZExtValue()) {
1492 return TrueIfSigned ? new ICmpInst(ICmpInst::ICMP_SLT, ShOp,
1494 : new ICmpInst(ICmpInst::ICMP_SGT, ShOp,
1496 }
1497
1498 return nullptr;
1499}
1500
1501/// Fold icmp (trunc X), (trunc Y).
1502/// Fold icmp (trunc X), (zext Y).
1505 const SimplifyQuery &Q) {
1506 if (Cmp.isSigned())
1507 return nullptr;
1508
1509 Value *X, *Y;
1511 bool YIsZext = false;
1512 // Try to match icmp (trunc X), (trunc Y)
1513 if (match(&Cmp, m_ICmp(Pred, m_Trunc(m_Value(X)), m_Trunc(m_Value(Y))))) {
1514 if (X->getType() != Y->getType() &&
1515 (!Cmp.getOperand(0)->hasOneUse() || !Cmp.getOperand(1)->hasOneUse()))
1516 return nullptr;
1517 if (!isDesirableIntType(X->getType()->getScalarSizeInBits()) &&
1518 isDesirableIntType(Y->getType()->getScalarSizeInBits())) {
1519 std::swap(X, Y);
1520 Pred = Cmp.getSwappedPredicate(Pred);
1521 }
1522 }
1523 // Try to match icmp (trunc X), (zext Y)
1524 else if (match(&Cmp, m_c_ICmp(Pred, m_Trunc(m_Value(X)),
1525 m_OneUse(m_ZExt(m_Value(Y))))))
1526
1527 YIsZext = true;
1528 else
1529 return nullptr;
1530
1531 Type *TruncTy = Cmp.getOperand(0)->getType();
1532 unsigned TruncBits = TruncTy->getScalarSizeInBits();
1533
1534 // If this transform will end up changing from desirable types -> undesirable
1535 // types skip it.
1536 if (isDesirableIntType(TruncBits) &&
1537 !isDesirableIntType(X->getType()->getScalarSizeInBits()))
1538 return nullptr;
1539
1540 // Check if the trunc is unneeded.
1541 KnownBits KnownX = llvm::computeKnownBits(X, /*Depth*/ 0, Q);
1542 if (KnownX.countMaxActiveBits() > TruncBits)
1543 return nullptr;
1544
1545 if (!YIsZext) {
1546 // If Y is also a trunc, make sure it is unneeded.
1547 KnownBits KnownY = llvm::computeKnownBits(Y, /*Depth*/ 0, Q);
1548 if (KnownY.countMaxActiveBits() > TruncBits)
1549 return nullptr;
1550 }
1551
1552 Value *NewY = Builder.CreateZExtOrTrunc(Y, X->getType());
1553 return new ICmpInst(Pred, X, NewY);
1554}
1555
1556/// Fold icmp (xor X, Y), C.
1559 const APInt &C) {
1560 if (Instruction *I = foldICmpXorShiftConst(Cmp, Xor, C))
1561 return I;
1562
1563 Value *X = Xor->getOperand(0);
1564 Value *Y = Xor->getOperand(1);
1565 const APInt *XorC;
1566 if (!match(Y, m_APInt(XorC)))
1567 return nullptr;
1568
1569 // If this is a comparison that tests the signbit (X < 0) or (x > -1),
1570 // fold the xor.
1571 ICmpInst::Predicate Pred = Cmp.getPredicate();
1572 bool TrueIfSigned = false;
1573 if (isSignBitCheck(Cmp.getPredicate(), C, TrueIfSigned)) {
1574
1575 // If the sign bit of the XorCst is not set, there is no change to
1576 // the operation, just stop using the Xor.
1577 if (!XorC->isNegative())
1578 return replaceOperand(Cmp, 0, X);
1579
1580 // Emit the opposite comparison.
1581 if (TrueIfSigned)
1582 return new ICmpInst(ICmpInst::ICMP_SGT, X,
1583 ConstantInt::getAllOnesValue(X->getType()));
1584 else
1585 return new ICmpInst(ICmpInst::ICMP_SLT, X,
1586 ConstantInt::getNullValue(X->getType()));
1587 }
1588
1589 if (Xor->hasOneUse()) {
1590 // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask))
1591 if (!Cmp.isEquality() && XorC->isSignMask()) {
1592 Pred = Cmp.getFlippedSignednessPredicate();
1593 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC));
1594 }
1595
1596 // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask))
1597 if (!Cmp.isEquality() && XorC->isMaxSignedValue()) {
1598 Pred = Cmp.getFlippedSignednessPredicate();
1599 Pred = Cmp.getSwappedPredicate(Pred);
1600 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC));
1601 }
1602 }
1603
1604 // Mask constant magic can eliminate an 'xor' with unsigned compares.
1605 if (Pred == ICmpInst::ICMP_UGT) {
1606 // (xor X, ~C) >u C --> X <u ~C (when C+1 is a power of 2)
1607 if (*XorC == ~C && (C + 1).isPowerOf2())
1608 return new ICmpInst(ICmpInst::ICMP_ULT, X, Y);
1609 // (xor X, C) >u C --> X >u C (when C+1 is a power of 2)
1610 if (*XorC == C && (C + 1).isPowerOf2())
1611 return new ICmpInst(ICmpInst::ICMP_UGT, X, Y);
1612 }
1613 if (Pred == ICmpInst::ICMP_ULT) {
1614 // (xor X, -C) <u C --> X >u ~C (when C is a power of 2)
1615 if (*XorC == -C && C.isPowerOf2())
1616 return new ICmpInst(ICmpInst::ICMP_UGT, X,
1617 ConstantInt::get(X->getType(), ~C));
1618 // (xor X, C) <u C --> X >u ~C (when -C is a power of 2)
1619 if (*XorC == C && (-C).isPowerOf2())
1620 return new ICmpInst(ICmpInst::ICMP_UGT, X,
1621 ConstantInt::get(X->getType(), ~C));
1622 }
1623 return nullptr;
1624}
1625
1626/// For power-of-2 C:
1627/// ((X s>> ShiftC) ^ X) u< C --> (X + C) u< (C << 1)
1628/// ((X s>> ShiftC) ^ X) u> (C - 1) --> (X + C) u> ((C << 1) - 1)
1631 const APInt &C) {
1632 CmpInst::Predicate Pred = Cmp.getPredicate();
1633 APInt PowerOf2;
1634 if (Pred == ICmpInst::ICMP_ULT)
1635 PowerOf2 = C;
1636 else if (Pred == ICmpInst::ICMP_UGT && !C.isMaxValue())
1637 PowerOf2 = C + 1;
1638 else
1639 return nullptr;
1640 if (!PowerOf2.isPowerOf2())
1641 return nullptr;
1642 Value *X;
1643 const APInt *ShiftC;
1645 m_AShr(m_Deferred(X), m_APInt(ShiftC))))))
1646 return nullptr;
1647 uint64_t Shift = ShiftC->getLimitedValue();
1648 Type *XType = X->getType();
1649 if (Shift == 0 || PowerOf2.isMinSignedValue())
1650 return nullptr;
1651 Value *Add = Builder.CreateAdd(X, ConstantInt::get(XType, PowerOf2));
1652 APInt Bound =
1653 Pred == ICmpInst::ICMP_ULT ? PowerOf2 << 1 : ((PowerOf2 << 1) - 1);
1654 return new ICmpInst(Pred, Add, ConstantInt::get(XType, Bound));
1655}
1656
1657/// Fold icmp (and (sh X, Y), C2), C1.
1660 const APInt &C1,
1661 const APInt &C2) {
1662 BinaryOperator *Shift = dyn_cast<BinaryOperator>(And->getOperand(0));
1663 if (!Shift || !Shift->isShift())
1664 return nullptr;
1665
1666 // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could
1667 // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in
1668 // code produced by the clang front-end, for bitfield access.
1669 // This seemingly simple opportunity to fold away a shift turns out to be
1670 // rather complicated. See PR17827 for details.
1671 unsigned ShiftOpcode = Shift->getOpcode();
1672 bool IsShl = ShiftOpcode == Instruction::Shl;
1673 const APInt *C3;
1674 if (match(Shift->getOperand(1), m_APInt(C3))) {
1675 APInt NewAndCst, NewCmpCst;
1676 bool AnyCmpCstBitsShiftedOut;
1677 if (ShiftOpcode == Instruction::Shl) {
1678 // For a left shift, we can fold if the comparison is not signed. We can
1679 // also fold a signed comparison if the mask value and comparison value
1680 // are not negative. These constraints may not be obvious, but we can
1681 // prove that they are correct using an SMT solver.
1682 if (Cmp.isSigned() && (C2.isNegative() || C1.isNegative()))
1683 return nullptr;
1684
1685 NewCmpCst = C1.lshr(*C3);
1686 NewAndCst = C2.lshr(*C3);
1687 AnyCmpCstBitsShiftedOut = NewCmpCst.shl(*C3) != C1;
1688 } else if (ShiftOpcode == Instruction::LShr) {
1689 // For a logical right shift, we can fold if the comparison is not signed.
1690 // We can also fold a signed comparison if the shifted mask value and the
1691 // shifted comparison value are not negative. These constraints may not be
1692 // obvious, but we can prove that they are correct using an SMT solver.
1693 NewCmpCst = C1.shl(*C3);
1694 NewAndCst = C2.shl(*C3);
1695 AnyCmpCstBitsShiftedOut = NewCmpCst.lshr(*C3) != C1;
1696 if (Cmp.isSigned() && (NewAndCst.isNegative() || NewCmpCst.isNegative()))
1697 return nullptr;
1698 } else {
1699 // For an arithmetic shift, check that both constants don't use (in a
1700 // signed sense) the top bits being shifted out.
1701 assert(ShiftOpcode == Instruction::AShr && "Unknown shift opcode");
1702 NewCmpCst = C1.shl(*C3);
1703 NewAndCst = C2.shl(*C3);
1704 AnyCmpCstBitsShiftedOut = NewCmpCst.ashr(*C3) != C1;
1705 if (NewAndCst.ashr(*C3) != C2)
1706 return nullptr;
1707 }
1708
1709 if (AnyCmpCstBitsShiftedOut) {
1710 // If we shifted bits out, the fold is not going to work out. As a
1711 // special case, check to see if this means that the result is always
1712 // true or false now.
1713 if (Cmp.getPredicate() == ICmpInst::ICMP_EQ)
1714 return replaceInstUsesWith(Cmp, ConstantInt::getFalse(Cmp.getType()));
1715 if (Cmp.getPredicate() == ICmpInst::ICMP_NE)
1716 return replaceInstUsesWith(Cmp, ConstantInt::getTrue(Cmp.getType()));
1717 } else {
1718 Value *NewAnd = Builder.CreateAnd(
1719 Shift->getOperand(0), ConstantInt::get(And->getType(), NewAndCst));
1720 return new ICmpInst(Cmp.getPredicate(),
1721 NewAnd, ConstantInt::get(And->getType(), NewCmpCst));
1722 }
1723 }
1724
1725 // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is
1726 // preferable because it allows the C2 << Y expression to be hoisted out of a
1727 // loop if Y is invariant and X is not.
1728 if (Shift->hasOneUse() && C1.isZero() && Cmp.isEquality() &&
1729 !Shift->isArithmeticShift() && !isa<Constant>(Shift->getOperand(0))) {
1730 // Compute C2 << Y.
1731 Value *NewShift =
1732 IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1))
1733 : Builder.CreateShl(And->getOperand(1), Shift->getOperand(1));
1734
1735 // Compute X & (C2 << Y).
1736 Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift);
1737 return replaceOperand(Cmp, 0, NewAnd);
1738 }
1739
1740 return nullptr;
1741}
1742
1743/// Fold icmp (and X, C2), C1.
1746 const APInt &C1) {
1747 bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE;
1748
1749 // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1
1750 // TODO: We canonicalize to the longer form for scalars because we have
1751 // better analysis/folds for icmp, and codegen may be better with icmp.
1752 if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isZero() &&
1753 match(And->getOperand(1), m_One()))
1754 return new TruncInst(And->getOperand(0), Cmp.getType());
1755
1756 const APInt *C2;
1757 Value *X;
1758 if (!match(And, m_And(m_Value(X), m_APInt(C2))))
1759 return nullptr;
1760
1761 // Don't perform the following transforms if the AND has multiple uses
1762 if (!And->hasOneUse())
1763 return nullptr;
1764
1765 if (Cmp.isEquality() && C1.isZero()) {
1766 // Restrict this fold to single-use 'and' (PR10267).
1767 // Replace (and X, (1 << size(X)-1) != 0) with X s< 0
1768 if (C2->isSignMask()) {
1769 Constant *Zero = Constant::getNullValue(X->getType());
1770 auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE;
1771 return new ICmpInst(NewPred, X, Zero);
1772 }
1773
1774 APInt NewC2 = *C2;
1775 KnownBits Know = computeKnownBits(And->getOperand(0), 0, And);
1776 // Set high zeros of C2 to allow matching negated power-of-2.
1777 NewC2 = *C2 | APInt::getHighBitsSet(C2->getBitWidth(),
1778 Know.countMinLeadingZeros());
1779
1780 // Restrict this fold only for single-use 'and' (PR10267).
1781 // ((%x & C) == 0) --> %x u< (-C) iff (-C) is power of two.
1782 if (NewC2.isNegatedPowerOf2()) {
1783 Constant *NegBOC = ConstantInt::get(And->getType(), -NewC2);
1784 auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT;
1785 return new ICmpInst(NewPred, X, NegBOC);
1786 }
1787 }
1788
1789 // If the LHS is an 'and' of a truncate and we can widen the and/compare to
1790 // the input width without changing the value produced, eliminate the cast:
1791 //
1792 // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1'
1793 //
1794 // We can do this transformation if the constants do not have their sign bits
1795 // set or if it is an equality comparison. Extending a relational comparison
1796 // when we're checking the sign bit would not work.
1797 Value *W;
1798 if (match(And->getOperand(0), m_OneUse(m_Trunc(m_Value(W)))) &&
1799 (Cmp.isEquality() || (!C1.isNegative() && !C2->isNegative()))) {
1800 // TODO: Is this a good transform for vectors? Wider types may reduce
1801 // throughput. Should this transform be limited (even for scalars) by using
1802 // shouldChangeType()?
1803 if (!Cmp.getType()->isVectorTy()) {
1804 Type *WideType = W->getType();
1805 unsigned WideScalarBits = WideType->getScalarSizeInBits();
1806 Constant *ZextC1 = ConstantInt::get(WideType, C1.zext(WideScalarBits));
1807 Constant *ZextC2 = ConstantInt::get(WideType, C2->zext(WideScalarBits));
1808 Value *NewAnd = Builder.CreateAnd(W, ZextC2, And->getName());
1809 return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1);
1810 }
1811 }
1812
1813 if (Instruction *I = foldICmpAndShift(Cmp, And, C1, *C2))
1814 return I;
1815
1816 // (icmp pred (and (or (lshr A, B), A), 1), 0) -->
1817 // (icmp pred (and A, (or (shl 1, B), 1), 0))
1818 //
1819 // iff pred isn't signed
1820 if (!Cmp.isSigned() && C1.isZero() && And->getOperand(0)->hasOneUse() &&
1821 match(And->getOperand(1), m_One())) {
1822 Constant *One = cast<Constant>(And->getOperand(1));
1823 Value *Or = And->getOperand(0);
1824 Value *A, *B, *LShr;
1825 if (match(Or, m_Or(m_Value(LShr), m_Value(A))) &&
1826 match(LShr, m_LShr(m_Specific(A), m_Value(B)))) {
1827 unsigned UsesRemoved = 0;
1828 if (And->hasOneUse())
1829 ++UsesRemoved;
1830 if (Or->hasOneUse())
1831 ++UsesRemoved;
1832 if (LShr->hasOneUse())
1833 ++UsesRemoved;
1834
1835 // Compute A & ((1 << B) | 1)
1836 unsigned RequireUsesRemoved = match(B, m_ImmConstant()) ? 1 : 3;
1837 if (UsesRemoved >= RequireUsesRemoved) {
1838 Value *NewOr =
1839 Builder.CreateOr(Builder.CreateShl(One, B, LShr->getName(),
1840 /*HasNUW=*/true),
1841 One, Or->getName());
1842 Value *NewAnd = Builder.CreateAnd(A, NewOr, And->getName());
1843 return replaceOperand(Cmp, 0, NewAnd);
1844 }
1845 }
1846 }
1847
1848 // (icmp eq (and (bitcast X to int), ExponentMask), ExponentMask) -->
1849 // llvm.is.fpclass(X, fcInf|fcNan)
1850 // (icmp ne (and (bitcast X to int), ExponentMask), ExponentMask) -->
1851 // llvm.is.fpclass(X, ~(fcInf|fcNan))
1852 Value *V;
1853 if (!Cmp.getParent()->getParent()->hasFnAttribute(
1854 Attribute::NoImplicitFloat) &&
1855 Cmp.isEquality() &&
1857 Type *FPType = V->getType()->getScalarType();
1858 if (FPType->isIEEELikeFPTy() && C1 == *C2) {
1859 APInt ExponentMask =
1861 if (C1 == ExponentMask) {
1862 unsigned Mask = FPClassTest::fcNan | FPClassTest::fcInf;
1863 if (isICMP_NE)
1864 Mask = ~Mask & fcAllFlags;
1865 return replaceInstUsesWith(Cmp, Builder.createIsFPClass(V, Mask));
1866 }
1867 }
1868 }
1869
1870 return nullptr;
1871}
1872
1873/// Fold icmp (and X, Y), C.
1876 const APInt &C) {
1877 if (Instruction *I = foldICmpAndConstConst(Cmp, And, C))
1878 return I;
1879
1880 const ICmpInst::Predicate Pred = Cmp.getPredicate();
1881 bool TrueIfNeg;
1882 if (isSignBitCheck(Pred, C, TrueIfNeg)) {
1883 // ((X - 1) & ~X) < 0 --> X == 0
1884 // ((X - 1) & ~X) >= 0 --> X != 0
1885 Value *X;
1886 if (match(And->getOperand(0), m_Add(m_Value(X), m_AllOnes())) &&
1887 match(And->getOperand(1), m_Not(m_Specific(X)))) {
1888 auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1889 return new ICmpInst(NewPred, X, ConstantInt::getNullValue(X->getType()));
1890 }
1891 // (X & -X) < 0 --> X == MinSignedC
1892 // (X & -X) > -1 --> X != MinSignedC
1893 if (match(And, m_c_And(m_Neg(m_Value(X)), m_Deferred(X)))) {
1894 Constant *MinSignedC = ConstantInt::get(
1895 X->getType(),
1896 APInt::getSignedMinValue(X->getType()->getScalarSizeInBits()));
1897 auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1898 return new ICmpInst(NewPred, X, MinSignedC);
1899 }
1900 }
1901
1902 // TODO: These all require that Y is constant too, so refactor with the above.
1903
1904 // Try to optimize things like "A[i] & 42 == 0" to index computations.
1905 Value *X = And->getOperand(0);
1906 Value *Y = And->getOperand(1);
1907 if (auto *C2 = dyn_cast<ConstantInt>(Y))
1908 if (auto *LI = dyn_cast<LoadInst>(X))
1909 if (auto *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0)))
1910 if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)))
1911 if (Instruction *Res =
1912 foldCmpLoadFromIndexedGlobal(LI, GEP, GV, Cmp, C2))
1913 return Res;
1914
1915 if (!Cmp.isEquality())
1916 return nullptr;
1917
1918 // X & -C == -C -> X > u ~C
1919 // X & -C != -C -> X <= u ~C
1920 // iff C is a power of 2
1921 if (Cmp.getOperand(1) == Y && C.isNegatedPowerOf2()) {
1922 auto NewPred =
1924 return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1))));
1925 }
1926
1927 // If we are testing the intersection of 2 select-of-nonzero-constants with no
1928 // common bits set, it's the same as checking if exactly one select condition
1929 // is set:
1930 // ((A ? TC : FC) & (B ? TC : FC)) == 0 --> xor A, B
1931 // ((A ? TC : FC) & (B ? TC : FC)) != 0 --> not(xor A, B)
1932 // TODO: Generalize for non-constant values.
1933 // TODO: Handle signed/unsigned predicates.
1934 // TODO: Handle other bitwise logic connectors.
1935 // TODO: Extend to handle a non-zero compare constant.
1936 if (C.isZero() && (Pred == CmpInst::ICMP_EQ || And->hasOneUse())) {
1937 assert(Cmp.isEquality() && "Not expecting non-equality predicates");
1938 Value *A, *B;
1939 const APInt *TC, *FC;
1940 if (match(X, m_Select(m_Value(A), m_APInt(TC), m_APInt(FC))) &&
1941 match(Y,
1942 m_Select(m_Value(B), m_SpecificInt(*TC), m_SpecificInt(*FC))) &&
1943 !TC->isZero() && !FC->isZero() && !TC->intersects(*FC)) {
1944 Value *R = Builder.CreateXor(A, B);
1945 if (Pred == CmpInst::ICMP_NE)
1946 R = Builder.CreateNot(R);
1947 return replaceInstUsesWith(Cmp, R);
1948 }
1949 }
1950
1951 // ((zext i1 X) & Y) == 0 --> !((trunc Y) & X)
1952 // ((zext i1 X) & Y) != 0 --> ((trunc Y) & X)
1953 // ((zext i1 X) & Y) == 1 --> ((trunc Y) & X)
1954 // ((zext i1 X) & Y) != 1 --> !((trunc Y) & X)
1956 X->getType()->isIntOrIntVectorTy(1) && (C.isZero() || C.isOne())) {
1957 Value *TruncY = Builder.CreateTrunc(Y, X->getType());
1958 if (C.isZero() ^ (Pred == CmpInst::ICMP_NE)) {
1959 Value *And = Builder.CreateAnd(TruncY, X);
1961 }
1962 return BinaryOperator::CreateAnd(TruncY, X);
1963 }
1964
1965 // (icmp eq/ne (and (shl -1, X), Y), 0)
1966 // -> (icmp eq/ne (lshr Y, X), 0)
1967 // We could technically handle any C == 0 or (C < 0 && isOdd(C)) but it seems
1968 // highly unlikely the non-zero case will ever show up in code.
1969 if (C.isZero() &&
1971 m_Value(Y))))) {
1972 Value *LShr = Builder.CreateLShr(Y, X);
1973 return new ICmpInst(Pred, LShr, Constant::getNullValue(LShr->getType()));
1974 }
1975
1976 return nullptr;
1977}
1978
1979/// Fold icmp eq/ne (or (xor/sub (X1, X2), xor/sub (X3, X4))), 0.
1981 InstCombiner::BuilderTy &Builder) {
1982 // Are we using xors or subs to bitwise check for a pair or pairs of
1983 // (in)equalities? Convert to a shorter form that has more potential to be
1984 // folded even further.
1985 // ((X1 ^/- X2) || (X3 ^/- X4)) == 0 --> (X1 == X2) && (X3 == X4)
1986 // ((X1 ^/- X2) || (X3 ^/- X4)) != 0 --> (X1 != X2) || (X3 != X4)
1987 // ((X1 ^/- X2) || (X3 ^/- X4) || (X5 ^/- X6)) == 0 -->
1988 // (X1 == X2) && (X3 == X4) && (X5 == X6)
1989 // ((X1 ^/- X2) || (X3 ^/- X4) || (X5 ^/- X6)) != 0 -->
1990 // (X1 != X2) || (X3 != X4) || (X5 != X6)
1992 SmallVector<Value *, 16> WorkList(1, Or);
1993
1994 while (!WorkList.empty()) {
1995 auto MatchOrOperatorArgument = [&](Value *OrOperatorArgument) {
1996 Value *Lhs, *Rhs;
1997
1998 if (match(OrOperatorArgument,
1999 m_OneUse(m_Xor(m_Value(Lhs), m_Value(Rhs))))) {
2000 CmpValues.emplace_back(Lhs, Rhs);
2001 return;
2002 }
2003
2004 if (match(OrOperatorArgument,
2005 m_OneUse(m_Sub(m_Value(Lhs), m_Value(Rhs))))) {
2006 CmpValues.emplace_back(Lhs, Rhs);
2007 return;
2008 }
2009
2010 WorkList.push_back(OrOperatorArgument);
2011 };
2012
2013 Value *CurrentValue = WorkList.pop_back_val();
2014 Value *OrOperatorLhs, *OrOperatorRhs;
2015
2016 if (!match(CurrentValue,
2017 m_Or(m_Value(OrOperatorLhs), m_Value(OrOperatorRhs)))) {
2018 return nullptr;
2019 }
2020
2021 MatchOrOperatorArgument(OrOperatorRhs);
2022 MatchOrOperatorArgument(OrOperatorLhs);
2023 }
2024
2025 ICmpInst::Predicate Pred = Cmp.getPredicate();
2026 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2027 Value *LhsCmp = Builder.CreateICmp(Pred, CmpValues.rbegin()->first,
2028 CmpValues.rbegin()->second);
2029
2030 for (auto It = CmpValues.rbegin() + 1; It != CmpValues.rend(); ++It) {
2031 Value *RhsCmp = Builder.CreateICmp(Pred, It->first, It->second);
2032 LhsCmp = Builder.CreateBinOp(BOpc, LhsCmp, RhsCmp);
2033 }
2034
2035 return LhsCmp;
2036}
2037
2038/// Fold icmp (or X, Y), C.
2041 const APInt &C) {
2042 ICmpInst::Predicate Pred = Cmp.getPredicate();
2043 if (C.isOne()) {
2044 // icmp slt signum(V) 1 --> icmp slt V, 1
2045 Value *V = nullptr;
2046 if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V))))
2047 return new ICmpInst(ICmpInst::ICMP_SLT, V,
2048 ConstantInt::get(V->getType(), 1));
2049 }
2050
2051 Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1);
2052 const APInt *MaskC;
2053 if (match(OrOp1, m_APInt(MaskC)) && Cmp.isEquality()) {
2054 if (*MaskC == C && (C + 1).isPowerOf2()) {
2055 // X | C == C --> X <=u C
2056 // X | C != C --> X >u C
2057 // iff C+1 is a power of 2 (C is a bitmask of the low bits)
2059 return new ICmpInst(Pred, OrOp0, OrOp1);
2060 }
2061
2062 // More general: canonicalize 'equality with set bits mask' to
2063 // 'equality with clear bits mask'.
2064 // (X | MaskC) == C --> (X & ~MaskC) == C ^ MaskC
2065 // (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC
2066 if (Or->hasOneUse()) {
2067 Value *And = Builder.CreateAnd(OrOp0, ~(*MaskC));
2068 Constant *NewC = ConstantInt::get(Or->getType(), C ^ (*MaskC));
2069 return new ICmpInst(Pred, And, NewC);
2070 }
2071 }
2072
2073 // (X | (X-1)) s< 0 --> X s< 1
2074 // (X | (X-1)) s> -1 --> X s> 0
2075 Value *X;
2076 bool TrueIfSigned;
2077 if (isSignBitCheck(Pred, C, TrueIfSigned) &&
2079 auto NewPred = TrueIfSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGT;
2080 Constant *NewC = ConstantInt::get(X->getType(), TrueIfSigned ? 1 : 0);
2081 return new ICmpInst(NewPred, X, NewC);
2082 }
2083
2084 const APInt *OrC;
2085 // icmp(X | OrC, C) --> icmp(X, 0)
2086 if (C.isNonNegative() && match(Or, m_Or(m_Value(X), m_APInt(OrC)))) {
2087 switch (Pred) {
2088 // X | OrC s< C --> X s< 0 iff OrC s>= C s>= 0
2089 case ICmpInst::ICMP_SLT:
2090 // X | OrC s>= C --> X s>= 0 iff OrC s>= C s>= 0
2091 case ICmpInst::ICMP_SGE:
2092 if (OrC->sge(C))
2093 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
2094 break;
2095 // X | OrC s<= C --> X s< 0 iff OrC s> C s>= 0
2096 case ICmpInst::ICMP_SLE:
2097 // X | OrC s> C --> X s>= 0 iff OrC s> C s>= 0
2098 case ICmpInst::ICMP_SGT:
2099 if (OrC->sgt(C))
2101 ConstantInt::getNullValue(X->getType()));
2102 break;
2103 default:
2104 break;
2105 }
2106 }
2107
2108 if (!Cmp.isEquality() || !C.isZero() || !Or->hasOneUse())
2109 return nullptr;
2110
2111 Value *P, *Q;
2113 // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0
2114 // -> and (icmp eq P, null), (icmp eq Q, null).
2115 Value *CmpP =
2116 Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType()));
2117 Value *CmpQ =
2119 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2120 return BinaryOperator::Create(BOpc, CmpP, CmpQ);
2121 }
2122
2123 if (Value *V = foldICmpOrXorSubChain(Cmp, Or, Builder))
2124 return replaceInstUsesWith(Cmp, V);
2125
2126 return nullptr;
2127}
2128
2129/// Fold icmp (mul X, Y), C.
2132 const APInt &C) {
2133 ICmpInst::Predicate Pred = Cmp.getPredicate();
2134 Type *MulTy = Mul->getType();
2135 Value *X = Mul->getOperand(0);
2136
2137 // If there's no overflow:
2138 // X * X == 0 --> X == 0
2139 // X * X != 0 --> X != 0
2140 if (Cmp.isEquality() && C.isZero() && X == Mul->getOperand(1) &&
2141 (Mul->hasNoUnsignedWrap() || Mul->hasNoSignedWrap()))
2142 return new ICmpInst(Pred, X, ConstantInt::getNullValue(MulTy));
2143
2144 const APInt *MulC;
2145 if (!match(Mul->getOperand(1), m_APInt(MulC)))
2146 return nullptr;
2147
2148 // If this is a test of the sign bit and the multiply is sign-preserving with
2149 // a constant operand, use the multiply LHS operand instead:
2150 // (X * +MulC) < 0 --> X < 0
2151 // (X * -MulC) < 0 --> X > 0
2152 if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) {
2153 if (MulC->isNegative())
2154 Pred = ICmpInst::getSwappedPredicate(Pred);
2155 return new ICmpInst(Pred, X, ConstantInt::getNullValue(MulTy));
2156 }
2157
2158 if (MulC->isZero())
2159 return nullptr;
2160
2161 // If the multiply does not wrap or the constant is odd, try to divide the
2162 // compare constant by the multiplication factor.
2163 if (Cmp.isEquality()) {
2164 // (mul nsw X, MulC) eq/ne C --> X eq/ne C /s MulC
2165 if (Mul->hasNoSignedWrap() && C.srem(*MulC).isZero()) {
2166 Constant *NewC = ConstantInt::get(MulTy, C.sdiv(*MulC));
2167 return new ICmpInst(Pred, X, NewC);
2168 }
2169
2170 // C % MulC == 0 is weaker than we could use if MulC is odd because it
2171 // correct to transform if MulC * N == C including overflow. I.e with i8
2172 // (icmp eq (mul X, 5), 101) -> (icmp eq X, 225) but since 101 % 5 != 0, we
2173 // miss that case.
2174 if (C.urem(*MulC).isZero()) {
2175 // (mul nuw X, MulC) eq/ne C --> X eq/ne C /u MulC
2176 // (mul X, OddC) eq/ne N * C --> X eq/ne N
2177 if ((*MulC & 1).isOne() || Mul->hasNoUnsignedWrap()) {
2178 Constant *NewC = ConstantInt::get(MulTy, C.udiv(*MulC));
2179 return new ICmpInst(Pred, X, NewC);
2180 }
2181 }
2182 }
2183
2184 // With a matching no-overflow guarantee, fold the constants:
2185 // (X * MulC) < C --> X < (C / MulC)
2186 // (X * MulC) > C --> X > (C / MulC)
2187 // TODO: Assert that Pred is not equal to SGE, SLE, UGE, ULE?
2188 Constant *NewC = nullptr;
2189 if (Mul->hasNoSignedWrap() && ICmpInst::isSigned(Pred)) {
2190 // MININT / -1 --> overflow.
2191 if (C.isMinSignedValue() && MulC->isAllOnes())
2192 return nullptr;
2193 if (MulC->isNegative())
2194 Pred = ICmpInst::getSwappedPredicate(Pred);
2195
2196 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) {
2197 NewC = ConstantInt::get(
2199 } else {
2200 assert((Pred == ICmpInst::ICMP_SLE || Pred == ICmpInst::ICMP_SGT) &&
2201 "Unexpected predicate");
2202 NewC = ConstantInt::get(
2204 }
2205 } else if (Mul->hasNoUnsignedWrap() && ICmpInst::isUnsigned(Pred)) {
2206 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) {
2207 NewC = ConstantInt::get(
2209 } else {
2210 assert((Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
2211 "Unexpected predicate");
2212 NewC = ConstantInt::get(
2214 }
2215 }
2216
2217 return NewC ? new ICmpInst(Pred, X, NewC) : nullptr;
2218}
2219
2220/// Fold icmp (shl 1, Y), C.
2222 const APInt &C) {
2223 Value *Y;
2224 if (!match(Shl, m_Shl(m_One(), m_Value(Y))))
2225 return nullptr;
2226
2227 Type *ShiftType = Shl->getType();
2228 unsigned TypeBits = C.getBitWidth();
2229 bool CIsPowerOf2 = C.isPowerOf2();
2230 ICmpInst::Predicate Pred = Cmp.getPredicate();
2231 if (Cmp.isUnsigned()) {
2232 // (1 << Y) pred C -> Y pred Log2(C)
2233 if (!CIsPowerOf2) {
2234 // (1 << Y) < 30 -> Y <= 4
2235 // (1 << Y) <= 30 -> Y <= 4
2236 // (1 << Y) >= 30 -> Y > 4
2237 // (1 << Y) > 30 -> Y > 4
2238 if (Pred == ICmpInst::ICMP_ULT)
2239 Pred = ICmpInst::ICMP_ULE;
2240 else if (Pred == ICmpInst::ICMP_UGE)
2241 Pred = ICmpInst::ICMP_UGT;
2242 }
2243
2244 unsigned CLog2 = C.logBase2();
2245 return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2));
2246 } else if (Cmp.isSigned()) {
2247 Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1);
2248 // (1 << Y) > 0 -> Y != 31
2249 // (1 << Y) > C -> Y != 31 if C is negative.
2250 if (Pred == ICmpInst::ICMP_SGT && C.sle(0))
2251 return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne);
2252
2253 // (1 << Y) < 0 -> Y == 31
2254 // (1 << Y) < 1 -> Y == 31
2255 // (1 << Y) < C -> Y == 31 if C is negative and not signed min.
2256 // Exclude signed min by subtracting 1 and lower the upper bound to 0.
2257 if (Pred == ICmpInst::ICMP_SLT && (C-1).sle(0))
2258 return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne);
2259 }
2260
2261 return nullptr;
2262}
2263
2264/// Fold icmp (shl X, Y), C.
2266 BinaryOperator *Shl,
2267 const APInt &C) {
2268 const APInt *ShiftVal;
2269 if (Cmp.isEquality() && match(Shl->getOperand(0), m_APInt(ShiftVal)))
2270 return foldICmpShlConstConst(Cmp, Shl->getOperand(1), C, *ShiftVal);
2271
2272 ICmpInst::Predicate Pred = Cmp.getPredicate();
2273 // (icmp pred (shl nuw&nsw X, Y), Csle0)
2274 // -> (icmp pred X, Csle0)
2275 //
2276 // The idea is the nuw/nsw essentially freeze the sign bit for the shift op
2277 // so X's must be what is used.
2278 if (C.sle(0) && Shl->hasNoUnsignedWrap() && Shl->hasNoSignedWrap())
2279 return new ICmpInst(Pred, Shl->getOperand(0), Cmp.getOperand(1));
2280
2281 // (icmp eq/ne (shl nuw|nsw X, Y), 0)
2282 // -> (icmp eq/ne X, 0)
2283 if (ICmpInst::isEquality(Pred) && C.isZero() &&
2284 (Shl->hasNoUnsignedWrap() || Shl->hasNoSignedWrap()))
2285 return new ICmpInst(Pred, Shl->getOperand(0), Cmp.getOperand(1));
2286
2287 // (icmp slt (shl nsw X, Y), 0/1)
2288 // -> (icmp slt X, 0/1)
2289 // (icmp sgt (shl nsw X, Y), 0/-1)
2290 // -> (icmp sgt X, 0/-1)
2291 //
2292 // NB: sge/sle with a constant will canonicalize to sgt/slt.
2293 if (Shl->hasNoSignedWrap() &&
2294 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT))
2295 if (C.isZero() || (Pred == ICmpInst::ICMP_SGT ? C.isAllOnes() : C.isOne()))
2296 return new ICmpInst(Pred, Shl->getOperand(0), Cmp.getOperand(1));
2297
2298 const APInt *ShiftAmt;
2299 if (!match(Shl->getOperand(1), m_APInt(ShiftAmt)))
2300 return foldICmpShlOne(Cmp, Shl, C);
2301
2302 // Check that the shift amount is in range. If not, don't perform undefined
2303 // shifts. When the shift is visited, it will be simplified.
2304 unsigned TypeBits = C.getBitWidth();
2305 if (ShiftAmt->uge(TypeBits))
2306 return nullptr;
2307
2308 Value *X = Shl->getOperand(0);
2309 Type *ShType = Shl->getType();
2310
2311 // NSW guarantees that we are only shifting out sign bits from the high bits,
2312 // so we can ASHR the compare constant without needing a mask and eliminate
2313 // the shift.
2314 if (Shl->hasNoSignedWrap()) {
2315 if (Pred == ICmpInst::ICMP_SGT) {
2316 // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt)
2317 APInt ShiftedC = C.ashr(*ShiftAmt);
2318 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2319 }
2320 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2321 C.ashr(*ShiftAmt).shl(*ShiftAmt) == C) {
2322 APInt ShiftedC = C.ashr(*ShiftAmt);
2323 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2324 }
2325 if (Pred == ICmpInst::ICMP_SLT) {
2326 // SLE is the same as above, but SLE is canonicalized to SLT, so convert:
2327 // (X << S) <=s C is equiv to X <=s (C >> S) for all C
2328 // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX
2329 // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN
2330 assert(!C.isMinSignedValue() && "Unexpected icmp slt");
2331 APInt ShiftedC = (C - 1).ashr(*ShiftAmt) + 1;
2332 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2333 }
2334 }
2335
2336 // NUW guarantees that we are only shifting out zero bits from the high bits,
2337 // so we can LSHR the compare constant without needing a mask and eliminate
2338 // the shift.
2339 if (Shl->hasNoUnsignedWrap()) {
2340 if (Pred == ICmpInst::ICMP_UGT) {
2341 // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt)
2342 APInt ShiftedC = C.lshr(*ShiftAmt);
2343 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2344 }
2345 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2346 C.lshr(*ShiftAmt).shl(*ShiftAmt) == C) {
2347 APInt ShiftedC = C.lshr(*ShiftAmt);
2348 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2349 }
2350 if (Pred == ICmpInst::ICMP_ULT) {
2351 // ULE is the same as above, but ULE is canonicalized to ULT, so convert:
2352 // (X << S) <=u C is equiv to X <=u (C >> S) for all C
2353 // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u
2354 // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0
2355 assert(C.ugt(0) && "ult 0 should have been eliminated");
2356 APInt ShiftedC = (C - 1).lshr(*ShiftAmt) + 1;
2357 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2358 }
2359 }
2360
2361 if (Cmp.isEquality() && Shl->hasOneUse()) {
2362 // Strength-reduce the shift into an 'and'.
2363 Constant *Mask = ConstantInt::get(
2364 ShType,
2365 APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt->getZExtValue()));
2366 Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask");
2367 Constant *LShrC = ConstantInt::get(ShType, C.lshr(*ShiftAmt));
2368 return new ICmpInst(Pred, And, LShrC);
2369 }
2370
2371 // Otherwise, if this is a comparison of the sign bit, simplify to and/test.
2372 bool TrueIfSigned = false;
2373 if (Shl->hasOneUse() && isSignBitCheck(Pred, C, TrueIfSigned)) {
2374 // (X << 31) <s 0 --> (X & 1) != 0
2375 Constant *Mask = ConstantInt::get(
2376 ShType,
2377 APInt::getOneBitSet(TypeBits, TypeBits - ShiftAmt->getZExtValue() - 1));
2378 Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask");
2379 return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ,
2380 And, Constant::getNullValue(ShType));
2381 }
2382
2383 // Simplify 'shl' inequality test into 'and' equality test.
2384 if (Cmp.isUnsigned() && Shl->hasOneUse()) {
2385 // (X l<< C2) u<=/u> C1 iff C1+1 is power of two -> X & (~C1 l>> C2) ==/!= 0
2386 if ((C + 1).isPowerOf2() &&
2387 (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)) {
2388 Value *And = Builder.CreateAnd(X, (~C).lshr(ShiftAmt->getZExtValue()));
2389 return new ICmpInst(Pred == ICmpInst::ICMP_ULE ? ICmpInst::ICMP_EQ
2391 And, Constant::getNullValue(ShType));
2392 }
2393 // (X l<< C2) u</u>= C1 iff C1 is power of two -> X & (-C1 l>> C2) ==/!= 0
2394 if (C.isPowerOf2() &&
2395 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) {
2396 Value *And =
2397 Builder.CreateAnd(X, (~(C - 1)).lshr(ShiftAmt->getZExtValue()));
2398 return new ICmpInst(Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_EQ
2400 And, Constant::getNullValue(ShType));
2401 }
2402 }
2403
2404 // Transform (icmp pred iM (shl iM %v, N), C)
2405 // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N))
2406 // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N.
2407 // This enables us to get rid of the shift in favor of a trunc that may be
2408 // free on the target. It has the additional benefit of comparing to a
2409 // smaller constant that may be more target-friendly.
2410 unsigned Amt = ShiftAmt->getLimitedValue(TypeBits - 1);
2411 if (Shl->hasOneUse() && Amt != 0 && C.countr_zero() >= Amt &&
2412 DL.isLegalInteger(TypeBits - Amt)) {
2413 Type *TruncTy = IntegerType::get(Cmp.getContext(), TypeBits - Amt);
2414 if (auto *ShVTy = dyn_cast<VectorType>(ShType))
2415 TruncTy = VectorType::get(TruncTy, ShVTy->getElementCount());
2416 Constant *NewC =
2417 ConstantInt::get(TruncTy, C.ashr(*ShiftAmt).trunc(TypeBits - Amt));
2418 return new ICmpInst(Pred, Builder.CreateTrunc(X, TruncTy), NewC);
2419 }
2420
2421 return nullptr;
2422}
2423
2424/// Fold icmp ({al}shr X, Y), C.
2426 BinaryOperator *Shr,
2427 const APInt &C) {
2428 // An exact shr only shifts out zero bits, so:
2429 // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0
2430 Value *X = Shr->getOperand(0);
2431 CmpInst::Predicate Pred = Cmp.getPredicate();
2432 if (Cmp.isEquality() && Shr->isExact() && C.isZero())
2433 return new ICmpInst(Pred, X, Cmp.getOperand(1));
2434
2435 bool IsAShr = Shr->getOpcode() == Instruction::AShr;
2436 const APInt *ShiftValC;
2437 if (match(X, m_APInt(ShiftValC))) {
2438 if (Cmp.isEquality())
2439 return foldICmpShrConstConst(Cmp, Shr->getOperand(1), C, *ShiftValC);
2440
2441 // (ShiftValC >> Y) >s -1 --> Y != 0 with ShiftValC < 0
2442 // (ShiftValC >> Y) <s 0 --> Y == 0 with ShiftValC < 0
2443 bool TrueIfSigned;
2444 if (!IsAShr && ShiftValC->isNegative() &&
2445 isSignBitCheck(Pred, C, TrueIfSigned))
2446 return new ICmpInst(TrueIfSigned ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE,
2447 Shr->getOperand(1),
2448 ConstantInt::getNullValue(X->getType()));
2449
2450 // If the shifted constant is a power-of-2, test the shift amount directly:
2451 // (ShiftValC >> Y) >u C --> X <u (LZ(C) - LZ(ShiftValC))
2452 // (ShiftValC >> Y) <u C --> X >=u (LZ(C-1) - LZ(ShiftValC))
2453 if (!IsAShr && ShiftValC->isPowerOf2() &&
2454 (Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_ULT)) {
2455 bool IsUGT = Pred == CmpInst::ICMP_UGT;
2456 assert(ShiftValC->uge(C) && "Expected simplify of compare");
2457 assert((IsUGT || !C.isZero()) && "Expected X u< 0 to simplify");
2458
2459 unsigned CmpLZ = IsUGT ? C.countl_zero() : (C - 1).countl_zero();
2460 unsigned ShiftLZ = ShiftValC->countl_zero();
2461 Constant *NewC = ConstantInt::get(Shr->getType(), CmpLZ - ShiftLZ);
2462 auto NewPred = IsUGT ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE;
2463 return new ICmpInst(NewPred, Shr->getOperand(1), NewC);
2464 }
2465 }
2466
2467 const APInt *ShiftAmtC;
2468 if (!match(Shr->getOperand(1), m_APInt(ShiftAmtC)))
2469 return nullptr;
2470
2471 // Check that the shift amount is in range. If not, don't perform undefined
2472 // shifts. When the shift is visited it will be simplified.
2473 unsigned TypeBits = C.getBitWidth();
2474 unsigned ShAmtVal = ShiftAmtC->getLimitedValue(TypeBits);
2475 if (ShAmtVal >= TypeBits || ShAmtVal == 0)
2476 return nullptr;
2477
2478 bool IsExact = Shr->isExact();
2479 Type *ShrTy = Shr->getType();
2480 // TODO: If we could guarantee that InstSimplify would handle all of the
2481 // constant-value-based preconditions in the folds below, then we could assert
2482 // those conditions rather than checking them. This is difficult because of
2483 // undef/poison (PR34838).
2484 if (IsAShr && Shr->hasOneUse()) {
2485 if (IsExact || Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) {
2486 // When ShAmtC can be shifted losslessly:
2487 // icmp PRED (ashr exact X, ShAmtC), C --> icmp PRED X, (C << ShAmtC)
2488 // icmp slt/ult (ashr X, ShAmtC), C --> icmp slt/ult X, (C << ShAmtC)
2489 APInt ShiftedC = C.shl(ShAmtVal);
2490 if (ShiftedC.ashr(ShAmtVal) == C)
2491 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2492 }
2493 if (Pred == CmpInst::ICMP_SGT) {
2494 // icmp sgt (ashr X, ShAmtC), C --> icmp sgt X, ((C + 1) << ShAmtC) - 1
2495 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2496 if (!C.isMaxSignedValue() && !(C + 1).shl(ShAmtVal).isMinSignedValue() &&
2497 (ShiftedC + 1).ashr(ShAmtVal) == (C + 1))
2498 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2499 }
2500 if (Pred == CmpInst::ICMP_UGT) {
2501 // icmp ugt (ashr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1
2502 // 'C + 1 << ShAmtC' can overflow as a signed number, so the 2nd
2503 // clause accounts for that pattern.
2504 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2505 if ((ShiftedC + 1).ashr(ShAmtVal) == (C + 1) ||
2506 (C + 1).shl(ShAmtVal).isMinSignedValue())
2507 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2508 }
2509
2510 // If the compare constant has significant bits above the lowest sign-bit,
2511 // then convert an unsigned cmp to a test of the sign-bit:
2512 // (ashr X, ShiftC) u> C --> X s< 0
2513 // (ashr X, ShiftC) u< C --> X s> -1
2514 if (C.getBitWidth() > 2 && C.getNumSignBits() <= ShAmtVal) {
2515 if (Pred == CmpInst::ICMP_UGT) {
2516 return new ICmpInst(CmpInst::ICMP_SLT, X,
2518 }
2519 if (Pred == CmpInst::ICMP_ULT) {
2520 return new ICmpInst(CmpInst::ICMP_SGT, X,
2522 }
2523 }
2524 } else if (!IsAShr) {
2525 if (Pred == CmpInst::ICMP_ULT || (Pred == CmpInst::ICMP_UGT && IsExact)) {
2526 // icmp ult (lshr X, ShAmtC), C --> icmp ult X, (C << ShAmtC)
2527 // icmp ugt (lshr exact X, ShAmtC), C --> icmp ugt X, (C << ShAmtC)
2528 APInt ShiftedC = C.shl(ShAmtVal);
2529 if (ShiftedC.lshr(ShAmtVal) == C)
2530 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2531 }
2532 if (Pred == CmpInst::ICMP_UGT) {
2533 // icmp ugt (lshr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1
2534 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2535 if ((ShiftedC + 1).lshr(ShAmtVal) == (C + 1))
2536 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2537 }
2538 }
2539
2540 if (!Cmp.isEquality())
2541 return nullptr;
2542
2543 // Handle equality comparisons of shift-by-constant.
2544
2545 // If the comparison constant changes with the shift, the comparison cannot
2546 // succeed (bits of the comparison constant cannot match the shifted value).
2547 // This should be known by InstSimplify and already be folded to true/false.
2548 assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) ||
2549 (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) &&
2550 "Expected icmp+shr simplify did not occur.");
2551
2552 // If the bits shifted out are known zero, compare the unshifted value:
2553 // (X & 4) >> 1 == 2 --> (X & 4) == 4.
2554 if (Shr->isExact())
2555 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, C << ShAmtVal));
2556
2557 if (C.isZero()) {
2558 // == 0 is u< 1.
2559 if (Pred == CmpInst::ICMP_EQ)
2560 return new ICmpInst(CmpInst::ICMP_ULT, X,
2561 ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal)));
2562 else
2563 return new ICmpInst(CmpInst::ICMP_UGT, X,
2564 ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal) - 1));
2565 }
2566
2567 if (Shr->hasOneUse()) {
2568 // Canonicalize the shift into an 'and':
2569 // icmp eq/ne (shr X, ShAmt), C --> icmp eq/ne (and X, HiMask), (C << ShAmt)
2570 APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal));
2571 Constant *Mask = ConstantInt::get(ShrTy, Val);
2572 Value *And = Builder.CreateAnd(X, Mask, Shr->getName() + ".mask");
2573 return new ICmpInst(Pred, And, ConstantInt::get(ShrTy, C << ShAmtVal));
2574 }
2575
2576 return nullptr;
2577}
2578
2580 BinaryOperator *SRem,
2581 const APInt &C) {
2582 // Match an 'is positive' or 'is negative' comparison of remainder by a
2583 // constant power-of-2 value:
2584 // (X % pow2C) sgt/slt 0
2585 const ICmpInst::Predicate Pred = Cmp.getPredicate();
2586 if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_SLT &&
2587 Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
2588 return nullptr;
2589
2590 // TODO: The one-use check is standard because we do not typically want to
2591 // create longer instruction sequences, but this might be a special-case
2592 // because srem is not good for analysis or codegen.
2593 if (!SRem->hasOneUse())
2594 return nullptr;
2595
2596 const APInt *DivisorC;
2597 if (!match(SRem->getOperand(1), m_Power2(DivisorC)))
2598 return nullptr;
2599
2600 // For cmp_sgt/cmp_slt only zero valued C is handled.
2601 // For cmp_eq/cmp_ne only positive valued C is handled.
2602 if (((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT) &&
2603 !C.isZero()) ||
2604 ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2605 !C.isStrictlyPositive()))
2606 return nullptr;
2607
2608 // Mask off the sign bit and the modulo bits (low-bits).
2609 Type *Ty = SRem->getType();
2611 Constant *MaskC = ConstantInt::get(Ty, SignMask | (*DivisorC - 1));
2612 Value *And = Builder.CreateAnd(SRem->getOperand(0), MaskC);
2613
2614 if (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE)
2615 return new ICmpInst(Pred, And, ConstantInt::get(Ty, C));
2616
2617 // For 'is positive?' check that the sign-bit is clear and at least 1 masked
2618 // bit is set. Example:
2619 // (i8 X % 32) s> 0 --> (X & 159) s> 0
2620 if (Pred == ICmpInst::ICMP_SGT)
2622
2623 // For 'is negative?' check that the sign-bit is set and at least 1 masked
2624 // bit is set. Example:
2625 // (i16 X % 4) s< 0 --> (X & 32771) u> 32768
2626 return new ICmpInst(ICmpInst::ICMP_UGT, And, ConstantInt::get(Ty, SignMask));
2627}
2628
2629/// Fold icmp (udiv X, Y), C.
2631 BinaryOperator *UDiv,
2632 const APInt &C) {
2633 ICmpInst::Predicate Pred = Cmp.getPredicate();
2634 Value *X = UDiv->getOperand(0);
2635 Value *Y = UDiv->getOperand(1);
2636 Type *Ty = UDiv->getType();
2637
2638 const APInt *C2;
2639 if (!match(X, m_APInt(C2)))
2640 return nullptr;
2641
2642 assert(*C2 != 0 && "udiv 0, X should have been simplified already.");
2643
2644 // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1))
2645 if (Pred == ICmpInst::ICMP_UGT) {
2646 assert(!C.isMaxValue() &&
2647 "icmp ugt X, UINT_MAX should have been simplified already.");
2648 return new ICmpInst(ICmpInst::ICMP_ULE, Y,
2649 ConstantInt::get(Ty, C2->udiv(C + 1)));
2650 }
2651
2652 // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C)
2653 if (Pred == ICmpInst::ICMP_ULT) {
2654 assert(C != 0 && "icmp ult X, 0 should have been simplified already.");
2655 return new ICmpInst(ICmpInst::ICMP_UGT, Y,
2656 ConstantInt::get(Ty, C2->udiv(C)));
2657 }
2658
2659 return nullptr;
2660}
2661
2662/// Fold icmp ({su}div X, Y), C.
2664 BinaryOperator *Div,
2665 const APInt &C) {
2666 ICmpInst::Predicate Pred = Cmp.getPredicate();
2667 Value *X = Div->getOperand(0);
2668 Value *Y = Div->getOperand(1);
2669 Type *Ty = Div->getType();
2670 bool DivIsSigned = Div->getOpcode() == Instruction::SDiv;
2671
2672 // If unsigned division and the compare constant is bigger than
2673 // UMAX/2 (negative), there's only one pair of values that satisfies an
2674 // equality check, so eliminate the division:
2675 // (X u/ Y) == C --> (X == C) && (Y == 1)
2676 // (X u/ Y) != C --> (X != C) || (Y != 1)
2677 // Similarly, if signed division and the compare constant is exactly SMIN:
2678 // (X s/ Y) == SMIN --> (X == SMIN) && (Y == 1)
2679 // (X s/ Y) != SMIN --> (X != SMIN) || (Y != 1)
2680 if (Cmp.isEquality() && Div->hasOneUse() && C.isSignBitSet() &&
2681 (!DivIsSigned || C.isMinSignedValue())) {
2682 Value *XBig = Builder.CreateICmp(Pred, X, ConstantInt::get(Ty, C));
2683 Value *YOne = Builder.CreateICmp(Pred, Y, ConstantInt::get(Ty, 1));
2684 auto Logic = Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2685 return BinaryOperator::Create(Logic, XBig, YOne);
2686 }
2687
2688 // Fold: icmp pred ([us]div X, C2), C -> range test
2689 // Fold this div into the comparison, producing a range check.
2690 // Determine, based on the divide type, what the range is being
2691 // checked. If there is an overflow on the low or high side, remember
2692 // it, otherwise compute the range [low, hi) bounding the new value.
2693 // See: InsertRangeTest above for the kinds of replacements possible.
2694 const APInt *C2;
2695 if (!match(Y, m_APInt(C2)))
2696 return nullptr;
2697
2698 // FIXME: If the operand types don't match the type of the divide
2699 // then don't attempt this transform. The code below doesn't have the
2700 // logic to deal with a signed divide and an unsigned compare (and
2701 // vice versa). This is because (x /s C2) <s C produces different
2702 // results than (x /s C2) <u C or (x /u C2) <s C or even
2703 // (x /u C2) <u C. Simply casting the operands and result won't
2704 // work. :( The if statement below tests that condition and bails
2705 // if it finds it.
2706 if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned())
2707 return nullptr;
2708
2709 // The ProdOV computation fails on divide by 0 and divide by -1. Cases with
2710 // INT_MIN will also fail if the divisor is 1. Although folds of all these
2711 // division-by-constant cases should be present, we can not assert that they
2712 // have happened before we reach this icmp instruction.
2713 if (C2->isZero() || C2->isOne() || (DivIsSigned && C2->isAllOnes()))
2714 return nullptr;
2715
2716 // Compute Prod = C * C2. We are essentially solving an equation of
2717 // form X / C2 = C. We solve for X by multiplying C2 and C.
2718 // By solving for X, we can turn this into a range check instead of computing
2719 // a divide.
2720 APInt Prod = C * *C2;
2721
2722 // Determine if the product overflows by seeing if the product is not equal to
2723 // the divide. Make sure we do the same kind of divide as in the LHS
2724 // instruction that we're folding.
2725 bool ProdOV = (DivIsSigned ? Prod.sdiv(*C2) : Prod.udiv(*C2)) != C;
2726
2727 // If the division is known to be exact, then there is no remainder from the
2728 // divide, so the covered range size is unit, otherwise it is the divisor.
2729 APInt RangeSize = Div->isExact() ? APInt(C2->getBitWidth(), 1) : *C2;
2730
2731 // Figure out the interval that is being checked. For example, a comparison
2732 // like "X /u 5 == 0" is really checking that X is in the interval [0, 5).
2733 // Compute this interval based on the constants involved and the signedness of
2734 // the compare/divide. This computes a half-open interval, keeping track of
2735 // whether either value in the interval overflows. After analysis each
2736 // overflow variable is set to 0 if it's corresponding bound variable is valid
2737 // -1 if overflowed off the bottom end, or +1 if overflowed off the top end.
2738 int LoOverflow = 0, HiOverflow = 0;
2739 APInt LoBound, HiBound;
2740
2741 if (!DivIsSigned) { // udiv
2742 // e.g. X/5 op 3 --> [15, 20)
2743 LoBound = Prod;
2744 HiOverflow = LoOverflow = ProdOV;
2745 if (!HiOverflow) {
2746 // If this is not an exact divide, then many values in the range collapse
2747 // to the same result value.
2748 HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false);
2749 }
2750 } else if (C2->isStrictlyPositive()) { // Divisor is > 0.
2751 if (C.isZero()) { // (X / pos) op 0
2752 // Can't overflow. e.g. X/2 op 0 --> [-1, 2)
2753 LoBound = -(RangeSize - 1);
2754 HiBound = RangeSize;
2755 } else if (C.isStrictlyPositive()) { // (X / pos) op pos
2756 LoBound = Prod; // e.g. X/5 op 3 --> [15, 20)
2757 HiOverflow = LoOverflow = ProdOV;
2758 if (!HiOverflow)
2759 HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true);
2760 } else { // (X / pos) op neg
2761 // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14)
2762 HiBound = Prod + 1;
2763 LoOverflow = HiOverflow = ProdOV ? -1 : 0;
2764 if (!LoOverflow) {
2765 APInt DivNeg = -RangeSize;
2766 LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0;
2767 }
2768 }
2769 } else if (C2->isNegative()) { // Divisor is < 0.
2770 if (Div->isExact())
2771 RangeSize.negate();
2772 if (C.isZero()) { // (X / neg) op 0
2773 // e.g. X/-5 op 0 --> [-4, 5)
2774 LoBound = RangeSize + 1;
2775 HiBound = -RangeSize;
2776 if (HiBound == *C2) { // -INTMIN = INTMIN
2777 HiOverflow = 1; // [INTMIN+1, overflow)
2778 HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN
2779 }
2780 } else if (C.isStrictlyPositive()) { // (X / neg) op pos
2781 // e.g. X/-5 op 3 --> [-19, -14)
2782 HiBound = Prod + 1;
2783 HiOverflow = LoOverflow = ProdOV ? -1 : 0;
2784 if (!LoOverflow)
2785 LoOverflow =
2786 addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1 : 0;
2787 } else { // (X / neg) op neg
2788 LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20)
2789 LoOverflow = HiOverflow = ProdOV;
2790 if (!HiOverflow)
2791 HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true);
2792 }
2793
2794 // Dividing by a negative swaps the condition. LT <-> GT
2795 Pred = ICmpInst::getSwappedPredicate(Pred);
2796 }
2797
2798 switch (Pred) {
2799 default:
2800 llvm_unreachable("Unhandled icmp predicate!");
2801 case ICmpInst::ICMP_EQ:
2802 if (LoOverflow && HiOverflow)
2803 return replaceInstUsesWith(Cmp, Builder.getFalse());
2804 if (HiOverflow)
2805 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
2806 X, ConstantInt::get(Ty, LoBound));
2807 if (LoOverflow)
2808 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
2809 X, ConstantInt::get(Ty, HiBound));
2810 return replaceInstUsesWith(
2811 Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, true));
2812 case ICmpInst::ICMP_NE:
2813 if (LoOverflow && HiOverflow)
2814 return replaceInstUsesWith(Cmp, Builder.getTrue());
2815 if (HiOverflow)
2816 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
2817 X, ConstantInt::get(Ty, LoBound));
2818 if (LoOverflow)
2819 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
2820 X, ConstantInt::get(Ty, HiBound));
2821 return replaceInstUsesWith(
2822 Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, false));
2823 case ICmpInst::ICMP_ULT:
2824 case ICmpInst::ICMP_SLT:
2825 if (LoOverflow == +1) // Low bound is greater than input range.
2826 return replaceInstUsesWith(Cmp, Builder.getTrue());
2827 if (LoOverflow == -1) // Low bound is less than input range.
2828 return replaceInstUsesWith(Cmp, Builder.getFalse());
2829 return new ICmpInst(Pred, X, ConstantInt::get(Ty, LoBound));
2830 case ICmpInst::ICMP_UGT:
2831 case ICmpInst::ICMP_SGT:
2832 if (HiOverflow == +1) // High bound greater than input range.
2833 return replaceInstUsesWith(Cmp, Builder.getFalse());
2834 if (HiOverflow == -1) // High bound less than input range.
2835 return replaceInstUsesWith(Cmp, Builder.getTrue());
2836 if (Pred == ICmpInst::ICMP_UGT)
2837 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, HiBound));
2838 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, HiBound));
2839 }
2840
2841 return nullptr;
2842}
2843
2844/// Fold icmp (sub X, Y), C.
2846 BinaryOperator *Sub,
2847 const APInt &C) {
2848 Value *X = Sub->getOperand(0), *Y = Sub->getOperand(1);
2849 ICmpInst::Predicate Pred = Cmp.getPredicate();
2850 Type *Ty = Sub->getType();
2851
2852 // (SubC - Y) == C) --> Y == (SubC - C)
2853 // (SubC - Y) != C) --> Y != (SubC - C)
2854 Constant *SubC;
2855 if (Cmp.isEquality() && match(X, m_ImmConstant(SubC))) {
2856 return new ICmpInst(Pred, Y,
2857 ConstantExpr::getSub(SubC, ConstantInt::get(Ty, C)));
2858 }
2859
2860 // (icmp P (sub nuw|nsw C2, Y), C) -> (icmp swap(P) Y, C2-C)
2861 const APInt *C2;
2862 APInt SubResult;
2863 ICmpInst::Predicate SwappedPred = Cmp.getSwappedPredicate();
2864 bool HasNSW = Sub->hasNoSignedWrap();
2865 bool HasNUW = Sub->hasNoUnsignedWrap();
2866 if (match(X, m_APInt(C2)) &&
2867 ((Cmp.isUnsigned() && HasNUW) || (Cmp.isSigned() && HasNSW)) &&
2868 !subWithOverflow(SubResult, *C2, C, Cmp.isSigned()))
2869 return new ICmpInst(SwappedPred, Y, ConstantInt::get(Ty, SubResult));
2870
2871 // X - Y == 0 --> X == Y.
2872 // X - Y != 0 --> X != Y.
2873 // TODO: We allow this with multiple uses as long as the other uses are not
2874 // in phis. The phi use check is guarding against a codegen regression
2875 // for a loop test. If the backend could undo this (and possibly
2876 // subsequent transforms), we would not need this hack.
2877 if (Cmp.isEquality() && C.isZero() &&
2878 none_of((Sub->users()), [](const User *U) { return isa<PHINode>(U); }))
2879 return new ICmpInst(Pred, X, Y);
2880
2881 // The following transforms are only worth it if the only user of the subtract
2882 // is the icmp.
2883 // TODO: This is an artificial restriction for all of the transforms below
2884 // that only need a single replacement icmp. Can these use the phi test
2885 // like the transform above here?
2886 if (!Sub->hasOneUse())
2887 return nullptr;
2888
2889 if (Sub->hasNoSignedWrap()) {
2890 // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y)
2891 if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
2892 return new ICmpInst(ICmpInst::ICMP_SGE, X, Y);
2893
2894 // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y)
2895 if (Pred == ICmpInst::ICMP_SGT && C.isZero())
2896 return new ICmpInst(ICmpInst::ICMP_SGT, X, Y);
2897
2898 // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y)
2899 if (Pred == ICmpInst::ICMP_SLT && C.isZero())
2900 return new ICmpInst(ICmpInst::ICMP_SLT, X, Y);
2901
2902 // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y)
2903 if (Pred == ICmpInst::ICMP_SLT && C.isOne())
2904 return new ICmpInst(ICmpInst::ICMP_SLE, X, Y);
2905 }
2906
2907 if (!match(X, m_APInt(C2)))
2908 return nullptr;
2909
2910 // C2 - Y <u C -> (Y | (C - 1)) == C2
2911 // iff (C2 & (C - 1)) == C - 1 and C is a power of 2
2912 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() &&
2913 (*C2 & (C - 1)) == (C - 1))
2914 return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(Y, C - 1), X);
2915
2916 // C2 - Y >u C -> (Y | C) != C2
2917 // iff C2 & C == C and C + 1 is a power of 2
2918 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == C)
2919 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(Y, C), X);
2920
2921 // We have handled special cases that reduce.
2922 // Canonicalize any remaining sub to add as:
2923 // (C2 - Y) > C --> (Y + ~C2) < ~C
2924 Value *Add = Builder.CreateAdd(Y, ConstantInt::get(Ty, ~(*C2)), "notsub",
2925 HasNUW, HasNSW);
2926 return new ICmpInst(SwappedPred, Add, ConstantInt::get(Ty, ~C));
2927}
2928
2929static Value *createLogicFromTable(const std::bitset<4> &Table, Value *Op0,
2930 Value *Op1, IRBuilderBase &Builder,
2931 bool HasOneUse) {
2932 auto FoldConstant = [&](bool Val) {
2933 Constant *Res = Val ? Builder.getTrue() : Builder.getFalse();
2934 if (Op0->getType()->isVectorTy())
2936 cast<VectorType>(Op0->getType())->getElementCount(), Res);
2937 return Res;
2938 };
2939
2940 switch (Table.to_ulong()) {
2941 case 0: // 0 0 0 0
2942 return FoldConstant(false);
2943 case 1: // 0 0 0 1
2944 return HasOneUse ? Builder.CreateNot(Builder.CreateOr(Op0, Op1)) : nullptr;
2945 case 2: // 0 0 1 0
2946 return HasOneUse ? Builder.CreateAnd(Builder.CreateNot(Op0), Op1) : nullptr;
2947 case 3: // 0 0 1 1
2948 return Builder.CreateNot(Op0);
2949 case 4: // 0 1 0 0
2950 return HasOneUse ? Builder.CreateAnd(Op0, Builder.CreateNot(Op1)) : nullptr;
2951 case 5: // 0 1 0 1
2952 return Builder.CreateNot(Op1);
2953 case 6: // 0 1 1 0
2954 return Builder.CreateXor(Op0, Op1);
2955 case 7: // 0 1 1 1
2956 return HasOneUse ? Builder.CreateNot(Builder.CreateAnd(Op0, Op1)) : nullptr;
2957 case 8: // 1 0 0 0
2958 return Builder.CreateAnd(Op0, Op1);
2959 case 9: // 1 0 0 1
2960 return HasOneUse ? Builder.CreateNot(Builder.CreateXor(Op0, Op1)) : nullptr;
2961 case 10: // 1 0 1 0
2962 return Op1;
2963 case 11: // 1 0 1 1
2964 return HasOneUse ? Builder.CreateOr(Builder.CreateNot(Op0), Op1) : nullptr;
2965 case 12: // 1 1 0 0
2966 return Op0;
2967 case 13: // 1 1 0 1
2968 return HasOneUse ? Builder.CreateOr(Op0, Builder.CreateNot(Op1)) : nullptr;
2969 case 14: // 1 1 1 0
2970 return Builder.CreateOr(Op0, Op1);
2971 case 15: // 1 1 1 1
2972 return FoldConstant(true);
2973 default:
2974 llvm_unreachable("Invalid Operation");
2975 }
2976 return nullptr;
2977}
2978
2979/// Fold icmp (add X, Y), C.
2982 const APInt &C) {
2983 Value *Y = Add->getOperand(1);
2984 Value *X = Add->getOperand(0);
2985
2986 Value *Op0, *Op1;
2987 Instruction *Ext0, *Ext1;
2988 const CmpInst::Predicate Pred = Cmp.getPredicate();
2989 if (match(Add,
2992 m_ZExtOrSExt(m_Value(Op1))))) &&
2993 Op0->getType()->isIntOrIntVectorTy(1) &&
2994 Op1->getType()->isIntOrIntVectorTy(1)) {
2995 unsigned BW = C.getBitWidth();
2996 std::bitset<4> Table;
2997 auto ComputeTable = [&](bool Op0Val, bool Op1Val) {
2998 int Res = 0;
2999 if (Op0Val)
3000 Res += isa<ZExtInst>(Ext0) ? 1 : -1;
3001 if (Op1Val)
3002 Res += isa<ZExtInst>(Ext1) ? 1 : -1;
3003 return ICmpInst::compare(APInt(BW, Res, true), C, Pred);
3004 };
3005
3006 Table[0] = ComputeTable(false, false);
3007 Table[1] = ComputeTable(false, true);
3008 Table[2] = ComputeTable(true, false);
3009 Table[3] = ComputeTable(true, true);
3010 if (auto *Cond =
3011 createLogicFromTable(Table, Op0, Op1, Builder, Add->hasOneUse()))
3012 return replaceInstUsesWith(Cmp, Cond);
3013 }
3014 const APInt *C2;
3015 if (Cmp.isEquality() || !match(Y, m_APInt(C2)))
3016 return nullptr;
3017
3018 // Fold icmp pred (add X, C2), C.
3019 Type *Ty = Add->getType();
3020
3021 // If the add does not wrap, we can always adjust the compare by subtracting
3022 // the constants. Equality comparisons are handled elsewhere. SGE/SLE/UGE/ULE
3023 // are canonicalized to SGT/SLT/UGT/ULT.
3024 if ((Add->hasNoSignedWrap() &&
3025 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) ||
3026 (Add->hasNoUnsignedWrap() &&
3027 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT))) {
3028 bool Overflow;
3029 APInt NewC =
3030 Cmp.isSigned() ? C.ssub_ov(*C2, Overflow) : C.usub_ov(*C2, Overflow);
3031 // If there is overflow, the result must be true or false.
3032 // TODO: Can we assert there is no overflow because InstSimplify always
3033 // handles those cases?
3034 if (!Overflow)
3035 // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2)
3036 return new ICmpInst(Pred, X, ConstantInt::get(Ty, NewC));
3037 }
3038
3039 auto CR = ConstantRange::makeExactICmpRegion(Pred, C).subtract(*C2);
3040 const APInt &Upper = CR.getUpper();
3041 const APInt &Lower = CR.getLower();
3042 if (Cmp.isSigned()) {
3043 if (Lower.isSignMask())
3044 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, Upper));
3045 if (Upper.isSignMask())
3046 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, Lower));
3047 } else {
3048 if (Lower.isMinValue())
3049 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, Upper));
3050 if (Upper.isMinValue())
3051 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, Lower));
3052 }
3053
3054 // This set of folds is intentionally placed after folds that use no-wrapping
3055 // flags because those folds are likely better for later analysis/codegen.
3058
3059 // Fold compare with offset to opposite sign compare if it eliminates offset:
3060 // (X + C2) >u C --> X <s -C2 (if C == C2 + SMAX)
3061 if (Pred == CmpInst::ICMP_UGT && C == *C2 + SMax)
3062 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, -(*C2)));
3063
3064 // (X + C2) <u C --> X >s ~C2 (if C == C2 + SMIN)
3065 if (Pred == CmpInst::ICMP_ULT && C == *C2 + SMin)
3066 return new ICmpInst(ICmpInst::ICMP_SGT, X, ConstantInt::get(Ty, ~(*C2)));
3067
3068 // (X + C2) >s C --> X <u (SMAX - C) (if C == C2 - 1)
3069 if (Pred == CmpInst::ICMP_SGT && C == *C2 - 1)
3070 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, SMax - C));
3071
3072 // (X + C2) <s C --> X >u (C ^ SMAX) (if C == C2)
3073 if (Pred == CmpInst::ICMP_SLT && C == *C2)
3074 return new ICmpInst(ICmpInst::ICMP_UGT, X, ConstantInt::get(Ty, C ^ SMax));
3075
3076 // (X + -1) <u C --> X <=u C (if X is never null)
3077 if (Pred == CmpInst::ICMP_ULT && C2->isAllOnes()) {
3078 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
3079 if (llvm::isKnownNonZero(X, DL, 0, Q.AC, Q.CxtI, Q.DT))
3080 return new ICmpInst(ICmpInst::ICMP_ULE, X, ConstantInt::get(Ty, C));
3081 }
3082
3083 if (!Add->hasOneUse())
3084 return nullptr;
3085
3086 // X+C <u C2 -> (X & -C2) == C
3087 // iff C & (C2-1) == 0
3088 // C2 is a power of 2
3089 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && (*C2 & (C - 1)) == 0)
3091 ConstantExpr::getNeg(cast<Constant>(Y)));
3092
3093 // X+C >u C2 -> (X & ~C2) != C
3094 // iff C & C2 == 0
3095 // C2+1 is a power of 2
3096 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == 0)
3098 ConstantExpr::getNeg(cast<Constant>(Y)));
3099
3100 // The range test idiom can use either ult or ugt. Arbitrarily canonicalize
3101 // to the ult form.
3102 // X+C2 >u C -> X+(C2-C-1) <u ~C
3103 if (Pred == ICmpInst::ICMP_UGT)
3104 return new ICmpInst(ICmpInst::ICMP_ULT,
3105 Builder.CreateAdd(X, ConstantInt::get(Ty, *C2 - C - 1)),
3106 ConstantInt::get(Ty, ~C));
3107
3108 return nullptr;
3109}
3110
3112 Value *&RHS, ConstantInt *&Less,
3113 ConstantInt *&Equal,
3114 ConstantInt *&Greater) {
3115 // TODO: Generalize this to work with other comparison idioms or ensure
3116 // they get canonicalized into this form.
3117
3118 // select i1 (a == b),
3119 // i32 Equal,
3120 // i32 (select i1 (a < b), i32 Less, i32 Greater)
3121 // where Equal, Less and Greater are placeholders for any three constants.
3122 ICmpInst::Predicate PredA;
3123 if (!match(SI->getCondition(), m_ICmp(PredA, m_Value(LHS), m_Value(RHS))) ||
3124 !ICmpInst::isEquality(PredA))
3125 return false;
3126 Value *EqualVal = SI->getTrueValue();
3127 Value *UnequalVal = SI->getFalseValue();
3128 // We still can get non-canonical predicate here, so canonicalize.
3129 if (PredA == ICmpInst::ICMP_NE)
3130 std::swap(EqualVal, UnequalVal);
3131 if (!match(EqualVal, m_ConstantInt(Equal)))
3132 return false;
3133 ICmpInst::Predicate PredB;
3134 Value *LHS2, *RHS2;
3135 if (!match(UnequalVal, m_Select(m_ICmp(PredB, m_Value(LHS2), m_Value(RHS2)),
3136 m_ConstantInt(Less), m_ConstantInt(Greater))))
3137 return false;
3138 // We can get predicate mismatch here, so canonicalize if possible:
3139 // First, ensure that 'LHS' match.
3140 if (LHS2 != LHS) {
3141 // x sgt y <--> y slt x
3142 std::swap(LHS2, RHS2);
3143 PredB = ICmpInst::getSwappedPredicate(PredB);
3144 }
3145 if (LHS2 != LHS)
3146 return false;
3147 // We also need to canonicalize 'RHS'.
3148 if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(RHS2)) {
3149 // x sgt C-1 <--> x sge C <--> not(x slt C)
3150 auto FlippedStrictness =
3152 PredB, cast<Constant>(RHS2));
3153 if (!FlippedStrictness)
3154 return false;
3155 assert(FlippedStrictness->first == ICmpInst::ICMP_SGE &&
3156 "basic correctness failure");
3157 RHS2 = FlippedStrictness->second;
3158 // And kind-of perform the result swap.
3159 std::swap(Less, Greater);
3160 PredB = ICmpInst::ICMP_SLT;
3161 }
3162 return PredB == ICmpInst::ICMP_SLT && RHS == RHS2;
3163}
3164
3167 ConstantInt *C) {
3168
3169 assert(C && "Cmp RHS should be a constant int!");
3170 // If we're testing a constant value against the result of a three way
3171 // comparison, the result can be expressed directly in terms of the
3172 // original values being compared. Note: We could possibly be more
3173 // aggressive here and remove the hasOneUse test. The original select is
3174 // really likely to simplify or sink when we remove a test of the result.
3175 Value *OrigLHS, *OrigRHS;
3176 ConstantInt *C1LessThan, *C2Equal, *C3GreaterThan;
3177 if (Cmp.hasOneUse() &&
3178 matchThreeWayIntCompare(Select, OrigLHS, OrigRHS, C1LessThan, C2Equal,
3179 C3GreaterThan)) {
3180 assert(C1LessThan && C2Equal && C3GreaterThan);
3181
3182 bool TrueWhenLessThan =
3183 ConstantExpr::getCompare(Cmp.getPredicate(), C1LessThan, C)
3184 ->isAllOnesValue();
3185 bool TrueWhenEqual =
3186 ConstantExpr::getCompare(Cmp.getPredicate(), C2Equal, C)
3187 ->isAllOnesValue();
3188 bool TrueWhenGreaterThan =
3189 ConstantExpr::getCompare(Cmp.getPredicate(), C3GreaterThan, C)
3190 ->isAllOnesValue();
3191
3192 // This generates the new instruction that will replace the original Cmp
3193 // Instruction. Instead of enumerating the various combinations when
3194 // TrueWhenLessThan, TrueWhenEqual and TrueWhenGreaterThan are true versus
3195 // false, we rely on chaining of ORs and future passes of InstCombine to
3196 // simplify the OR further (i.e. a s< b || a == b becomes a s<= b).
3197
3198 // When none of the three constants satisfy the predicate for the RHS (C),
3199 // the entire original Cmp can be simplified to a false.
3201 if (TrueWhenLessThan)
3203 OrigLHS, OrigRHS));
3204 if (TrueWhenEqual)
3206 OrigLHS, OrigRHS));
3207 if (TrueWhenGreaterThan)
3209 OrigLHS, OrigRHS));
3210
3211 return replaceInstUsesWith(Cmp, Cond);
3212 }
3213 return nullptr;
3214}
3215
3217 auto *Bitcast = dyn_cast<BitCastInst>(Cmp.getOperand(0));
3218 if (!Bitcast)
3219 return nullptr;
3220
3221 ICmpInst::Predicate Pred = Cmp.getPredicate();
3222 Value *Op1 = Cmp.getOperand(1);
3223 Value *BCSrcOp = Bitcast->getOperand(0);
3224 Type *SrcType = Bitcast->getSrcTy();
3225 Type *DstType = Bitcast->getType();
3226
3227 // Make sure the bitcast doesn't change between scalar and vector and
3228 // doesn't change the number of vector elements.
3229 if (SrcType->isVectorTy() == DstType->isVectorTy() &&
3230 SrcType->getScalarSizeInBits() == DstType->getScalarSizeInBits()) {
3231 // Zero-equality and sign-bit checks are preserved through sitofp + bitcast.
3232 Value *X;
3233 if (match(BCSrcOp, m_SIToFP(m_Value(X)))) {
3234 // icmp eq (bitcast (sitofp X)), 0 --> icmp eq X, 0
3235 // icmp ne (bitcast (sitofp X)), 0 --> icmp ne X, 0
3236 // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0
3237 // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0
3238 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT ||
3239 Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) &&
3240 match(Op1, m_Zero()))
3241 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
3242
3243 // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1
3244 if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_One()))
3245 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), 1));
3246
3247 // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1
3248 if (Pred == ICmpInst::ICMP_SGT && match(Op1, m_AllOnes()))
3249 return new ICmpInst(Pred, X,
3250 ConstantInt::getAllOnesValue(X->getType()));
3251 }
3252
3253 // Zero-equality checks are preserved through unsigned floating-point casts:
3254 // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0
3255 // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0
3256 if (match(BCSrcOp, m_UIToFP(m_Value(X))))
3257 if (Cmp.isEquality() && match(Op1, m_Zero()))
3258 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
3259
3260 const APInt *C;
3261 bool TrueIfSigned;
3262 if (match(Op1, m_APInt(C)) && Bitcast->hasOneUse()) {
3263 // If this is a sign-bit test of a bitcast of a casted FP value, eliminate
3264 // the FP extend/truncate because that cast does not change the sign-bit.
3265 // This is true for all standard IEEE-754 types and the X86 80-bit type.
3266 // The sign-bit is always the most significant bit in those types.
3267 if (isSignBitCheck(Pred, *C, TrueIfSigned) &&
3268 (match(BCSrcOp, m_FPExt(m_Value(X))) ||
3269 match(BCSrcOp, m_FPTrunc(m_Value(X))))) {
3270 // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0
3271 // (bitcast (fpext/fptrunc X)) to iX) > -1 --> (bitcast X to iY) > -1
3272 Type *XType = X->getType();
3273
3274 // We can't currently handle Power style floating point operations here.
3275 if (!(XType->isPPC_FP128Ty() || SrcType->isPPC_FP128Ty())) {
3276 Type *NewType = Builder.getIntNTy(XType->getScalarSizeInBits());
3277 if (auto *XVTy = dyn_cast<VectorType>(XType))
3278 NewType = VectorType::get(NewType, XVTy->getElementCount());
3279 Value *NewBitcast = Builder.CreateBitCast(X, NewType);
3280 if (TrueIfSigned)
3281 return new ICmpInst(ICmpInst::ICMP_SLT, NewBitcast,
3282 ConstantInt::getNullValue(NewType));
3283 else
3284 return new ICmpInst(ICmpInst::ICMP_SGT, NewBitcast,
3286 }
3287 }
3288
3289 // icmp eq/ne (bitcast X to int), special fp -> llvm.is.fpclass(X, class)
3290 Type *FPType = SrcType->getScalarType();
3291 if (!Cmp.getParent()->getParent()->hasFnAttribute(
3292 Attribute::NoImplicitFloat) &&
3293 Cmp.isEquality() && FPType->isIEEELikeFPTy()) {
3294 FPClassTest Mask = APFloat(FPType->getFltSemantics(), *C).classify();
3295 if (Mask & (fcInf | fcZero)) {
3296 if (Pred == ICmpInst::ICMP_NE)
3297 Mask = ~Mask;
3298 return replaceInstUsesWith(Cmp,
3299 Builder.createIsFPClass(BCSrcOp, Mask));
3300 }
3301 }
3302 }
3303 }
3304
3305 const APInt *C;
3306 if (!match(Cmp.getOperand(1), m_APInt(C)) || !DstType->isIntegerTy() ||
3307 !SrcType->isIntOrIntVectorTy())
3308 return nullptr;
3309
3310 // If this is checking if all elements of a vector compare are set or not,
3311 // invert the casted vector equality compare and test if all compare
3312 // elements are clear or not. Compare against zero is generally easier for
3313 // analysis and codegen.
3314 // icmp eq/ne (bitcast (not X) to iN), -1 --> icmp eq/ne (bitcast X to iN), 0
3315 // Example: are all elements equal? --> are zero elements not equal?
3316 // TODO: Try harder to reduce compare of 2 freely invertible operands?
3317 if (Cmp.isEquality() && C->isAllOnes() && Bitcast->hasOneUse()) {
3318 if (Value *NotBCSrcOp =
3319 getFreelyInverted(BCSrcOp, BCSrcOp->hasOneUse(), &Builder)) {
3320 Value *Cast = Builder.CreateBitCast(NotBCSrcOp, DstType);
3321 return new ICmpInst(Pred, Cast, ConstantInt::getNullValue(DstType));
3322 }
3323 }
3324
3325 // If this is checking if all elements of an extended vector are clear or not,
3326 // compare in a narrow type to eliminate the extend:
3327 // icmp eq/ne (bitcast (ext X) to iN), 0 --> icmp eq/ne (bitcast X to iM), 0
3328 Value *X;
3329 if (Cmp.isEquality() && C->isZero() && Bitcast->hasOneUse() &&
3330 match(BCSrcOp, m_ZExtOrSExt(m_Value(X)))) {
3331 if (auto *VecTy = dyn_cast<FixedVectorType>(X->getType())) {
3332 Type *NewType = Builder.getIntNTy(VecTy->getPrimitiveSizeInBits());
3333 Value *NewCast = Builder.CreateBitCast(X, NewType);
3334 return new ICmpInst(Pred, NewCast, ConstantInt::getNullValue(NewType));
3335 }
3336 }
3337
3338 // Folding: icmp <pred> iN X, C
3339 // where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN
3340 // and C is a splat of a K-bit pattern
3341 // and SC is a constant vector = <C', C', C', ..., C'>
3342 // Into:
3343 // %E = extractelement <M x iK> %vec, i32 C'
3344 // icmp <pred> iK %E, trunc(C)
3345 Value *Vec;
3346 ArrayRef<int> Mask;
3347 if (match(BCSrcOp, m_Shuffle(m_Value(Vec), m_Undef(), m_Mask(Mask)))) {
3348 // Check whether every element of Mask is the same constant
3349 if (all_equal(Mask)) {
3350 auto *VecTy = cast<VectorType>(SrcType);
3351 auto *EltTy = cast<IntegerType>(VecTy->getElementType());
3352 if (C->isSplat(EltTy->getBitWidth())) {
3353 // Fold the icmp based on the value of C
3354 // If C is M copies of an iK sized bit pattern,
3355 // then:
3356 // => %E = extractelement <N x iK> %vec, i32 Elem
3357 // icmp <pred> iK %SplatVal, <pattern>
3358 Value *Elem = Builder.getInt32(Mask[0]);
3359 Value *Extract = Builder.CreateExtractElement(Vec, Elem);
3360 Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth()));
3361 return new ICmpInst(Pred, Extract, NewC);
3362 }
3363 }
3364 }
3365 return nullptr;
3366}
3367
3368/// Try to fold integer comparisons with a constant operand: icmp Pred X, C
3369/// where X is some kind of instruction.
3371 const APInt *C;
3372
3373 if (match(Cmp.getOperand(1), m_APInt(C))) {
3374 if (auto *BO = dyn_cast<BinaryOperator>(Cmp.getOperand(0)))
3375 if (Instruction *I = foldICmpBinOpWithConstant(Cmp, BO, *C))
3376 return I;
3377
3378 if (auto *SI = dyn_cast<SelectInst>(Cmp.getOperand(0)))
3379 // For now, we only support constant integers while folding the
3380 // ICMP(SELECT)) pattern. We can extend this to support vector of integers
3381 // similar to the cases handled by binary ops above.
3382 if (auto *ConstRHS = dyn_cast<ConstantInt>(Cmp.getOperand(1)))
3383 if (Instruction *I = foldICmpSelectConstant(Cmp, SI, ConstRHS))
3384 return I;
3385
3386 if (auto *TI = dyn_cast<TruncInst>(Cmp.getOperand(0)))
3387 if (Instruction *I = foldICmpTruncConstant(Cmp, TI, *C))
3388 return I;
3389
3390 if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0)))
3391 if (Instruction *I = foldICmpIntrinsicWithConstant(Cmp, II, *C))
3392 return I;
3393
3394 // (extractval ([s/u]subo X, Y), 0) == 0 --> X == Y
3395 // (extractval ([s/u]subo X, Y), 0) != 0 --> X != Y
3396 // TODO: This checks one-use, but that is not strictly necessary.
3397 Value *Cmp0 = Cmp.getOperand(0);
3398 Value *X, *Y;
3399 if (C->isZero() && Cmp.isEquality() && Cmp0->hasOneUse() &&
3400 (match(Cmp0,
3401 m_ExtractValue<0>(m_Intrinsic<Intrinsic::ssub_with_overflow>(
3402 m_Value(X), m_Value(Y)))) ||
3403 match(Cmp0,
3404 m_ExtractValue<0>(m_Intrinsic<Intrinsic::usub_with_overflow>(
3405 m_Value(X), m_Value(Y))))))
3406 return new ICmpInst(Cmp.getPredicate(), X, Y);
3407 }
3408
3409 if (match(Cmp.getOperand(1), m_APIntAllowUndef(C)))
3411
3412 return nullptr;
3413}
3414
3415/// Fold an icmp equality instruction with binary operator LHS and constant RHS:
3416/// icmp eq/ne BO, C.
3418 ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) {
3419 // TODO: Some of these folds could work with arbitrary constants, but this
3420 // function is limited to scalar and vector splat constants.
3421 if (!Cmp.isEquality())
3422 return nullptr;
3423
3424 ICmpInst::Predicate Pred = Cmp.getPredicate();
3425 bool isICMP_NE = Pred == ICmpInst::ICMP_NE;
3426 Constant *RHS = cast<Constant>(Cmp.getOperand(1));
3427 Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1);
3428
3429 switch (BO->getOpcode()) {
3430 case Instruction::SRem:
3431 // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one.
3432 if (C.isZero() && BO->hasOneUse()) {
3433 const APInt *BOC;
3434 if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) {
3435 Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName());
3436 return new ICmpInst(Pred, NewRem,
3438 }
3439 }
3440 break;
3441 case Instruction::Add: {
3442 // (A + C2) == C --> A == (C - C2)
3443 // (A + C2) != C --> A != (C - C2)
3444 // TODO: Remove the one-use limitation? See discussion in D58633.
3445 if (Constant *C2 = dyn_cast<Constant>(BOp1)) {
3446 if (BO->hasOneUse())
3447 return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(RHS, C2));
3448 } else if (C.isZero()) {
3449 // Replace ((add A, B) != 0) with (A != -B) if A or B is
3450 // efficiently invertible, or if the add has just this one use.
3451 if (Value *NegVal = dyn_castNegVal(BOp1))
3452 return new ICmpInst(Pred, BOp0, NegVal);
3453 if (Value *NegVal = dyn_castNegVal(BOp0))
3454 return new ICmpInst(Pred, NegVal, BOp1);
3455 if (BO->hasOneUse()) {
3456 Value *Neg = Builder.CreateNeg(BOp1);
3457 Neg->takeName(BO);
3458 return new ICmpInst(Pred, BOp0, Neg);
3459 }
3460 }
3461 break;
3462 }
3463 case Instruction::Xor:
3464 if (BO->hasOneUse()) {
3465 if (Constant *BOC = dyn_cast<Constant>(BOp1)) {
3466 // For the xor case, we can xor two constants together, eliminating
3467 // the explicit xor.
3468 return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC));
3469 } else if (C.isZero()) {
3470 // Replace ((xor A, B) != 0) with (A != B)
3471 return new ICmpInst(Pred, BOp0, BOp1);
3472 }
3473 }
3474 break;
3475 case Instruction::Or: {
3476 const APInt *BOC;
3477 if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) {
3478 // Comparing if all bits outside of a constant mask are set?
3479 // Replace (X | C) == -1 with (X & ~C) == ~C.
3480 // This removes the -1 constant.
3481 Constant *NotBOC = ConstantExpr::getNot(cast<Constant>(BOp1));
3482 Value *And = Builder.CreateAnd(BOp0, NotBOC);
3483 return new ICmpInst(Pred, And, NotBOC);
3484 }
3485 break;
3486 }
3487 case Instruction::UDiv:
3488 case Instruction::SDiv:
3489 if (BO->isExact()) {
3490 // div exact X, Y eq/ne 0 -> X eq/ne 0
3491 // div exact X, Y eq/ne 1 -> X eq/ne Y
3492 // div exact X, Y eq/ne C ->
3493 // if Y * C never-overflow && OneUse:
3494 // -> Y * C eq/ne X
3495 if (C.isZero())
3496 return new ICmpInst(Pred, BOp0, Constant::getNullValue(BO->getType()));
3497 else if (C.isOne())
3498 return new ICmpInst(Pred, BOp0, BOp1);
3499 else if (BO->hasOneUse()) {
3501 Instruction::Mul, BO->getOpcode() == Instruction::SDiv, BOp1,
3502 Cmp.getOperand(1), BO);
3504 Value *YC =
3505 Builder.CreateMul(BOp1, ConstantInt::get(BO->getType(), C));
3506 return new ICmpInst(Pred, YC, BOp0);
3507 }
3508 }
3509 }
3510 if (BO->getOpcode() == Instruction::UDiv && C.isZero()) {
3511 // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A)
3512 auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT;
3513 return new ICmpInst(NewPred, BOp1, BOp0);
3514 }
3515 break;
3516 default:
3517 break;
3518 }
3519 return nullptr;
3520}
3521
3523 const APInt &CRhs,
3524 InstCombiner::BuilderTy &Builder,
3525 const SimplifyQuery &Q) {
3526 assert(CtpopLhs->getIntrinsicID() == Intrinsic::ctpop &&
3527 "Non-ctpop intrin in ctpop fold");
3528 if (!CtpopLhs->hasOneUse())
3529 return nullptr;
3530
3531 // Power of 2 test:
3532 // isPow2OrZero : ctpop(X) u< 2
3533 // isPow2 : ctpop(X) == 1
3534 // NotPow2OrZero: ctpop(X) u> 1
3535 // NotPow2 : ctpop(X) != 1
3536 // If we know any bit of X can be folded to:
3537 // IsPow2 : X & (~Bit) == 0
3538 // NotPow2 : X & (~Bit) != 0
3539 const ICmpInst::Predicate Pred = I.getPredicate();
3540 if (((I.isEquality() || Pred == ICmpInst::ICMP_UGT) && CRhs == 1) ||
3541 (Pred == ICmpInst::ICMP_ULT && CRhs == 2)) {
3542 Value *Op = CtpopLhs->getArgOperand(0);
3543 KnownBits OpKnown = computeKnownBits(Op, Q.DL,
3544 /*Depth*/ 0, Q.AC, Q.CxtI, Q.DT);
3545 // No need to check for count > 1, that should be already constant folded.
3546 if (OpKnown.countMinPopulation() == 1) {
3547 Value *And = Builder.CreateAnd(
3548 Op, Constant::getIntegerValue(Op->getType(), ~(OpKnown.One)));
3549 return new ICmpInst(
3550 (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_ULT)
3553 And, Constant::getNullValue(Op->getType()));
3554 }
3555 }
3556
3557 return nullptr;
3558}
3559
3560/// Fold an equality icmp with LLVM intrinsic and constant operand.
3562 ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) {
3563 Type *Ty = II->getType();
3564 unsigned BitWidth = C.getBitWidth();
3565 const ICmpInst::Predicate Pred = Cmp.getPredicate();
3566
3567 switch (II->getIntrinsicID()) {
3568 case Intrinsic::abs:
3569 // abs(A) == 0 -> A == 0
3570 // abs(A) == INT_MIN -> A == INT_MIN
3571 if (C.isZero() || C.isMinSignedValue())
3572 return new ICmpInst(Pred, II->getArgOperand(0), ConstantInt::get(Ty, C));
3573 break;
3574
3575 case Intrinsic::bswap:
3576 // bswap(A) == C -> A == bswap(C)
3577 return new ICmpInst(Pred, II->getArgOperand(0),
3578 ConstantInt::get(Ty, C.byteSwap()));
3579
3580 case Intrinsic::bitreverse:
3581 // bitreverse(A) == C -> A == bitreverse(C)
3582 return new ICmpInst(Pred, II->getArgOperand(0),
3583 ConstantInt::get(Ty, C.reverseBits()));
3584
3585 case Intrinsic::ctlz:
3586 case Intrinsic::cttz: {
3587 // ctz(A) == bitwidth(A) -> A == 0 and likewise for !=
3588 if (C == BitWidth)
3589 return new ICmpInst(Pred, II->getArgOperand(0),
3591
3592 // ctz(A) == C -> A & Mask1 == Mask2, where Mask2 only has bit C set
3593 // and Mask1 has bits 0..C+1 set. Similar for ctl, but for high bits.
3594 // Limit to one use to ensure we don't increase instruction count.
3595 unsigned Num = C.getLimitedValue(BitWidth);
3596 if (Num != BitWidth && II->hasOneUse()) {
3597 bool IsTrailing = II->getIntrinsicID() == Intrinsic::cttz;
3598 APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(BitWidth, Num + 1)
3599 : APInt::getHighBitsSet(BitWidth, Num + 1);
3600 APInt Mask2 = IsTrailing
3603 return new ICmpInst(Pred, Builder.CreateAnd(II->getArgOperand(0), Mask1),
3604 ConstantInt::get(Ty, Mask2));
3605 }
3606 break;
3607 }
3608
3609 case Intrinsic::ctpop: {
3610 // popcount(A) == 0 -> A == 0 and likewise for !=
3611 // popcount(A) == bitwidth(A) -> A == -1 and likewise for !=
3612 bool IsZero = C.isZero();
3613 if (IsZero || C == BitWidth)
3614 return new ICmpInst(Pred, II->getArgOperand(0),
3615 IsZero ? Constant::getNullValue(Ty)
3617
3618 break;
3619 }
3620
3621 case Intrinsic::fshl:
3622 case Intrinsic::fshr:
3623 if (II->getArgOperand(0) == II->getArgOperand(1)) {
3624 const APInt *RotAmtC;
3625 // ror(X, RotAmtC) == C --> X == rol(C, RotAmtC)
3626 // rol(X, RotAmtC) == C --> X == ror(C, RotAmtC)
3627 if (match(II->getArgOperand(2), m_APInt(RotAmtC)))
3628 return new ICmpInst(Pred, II->getArgOperand(0),
3629 II->getIntrinsicID() == Intrinsic::fshl
3630 ? ConstantInt::get(Ty, C.rotr(*RotAmtC))
3631 : ConstantInt::get(Ty, C.rotl(*RotAmtC)));
3632 }
3633 break;
3634
3635 case Intrinsic::umax:
3636 case Intrinsic::uadd_sat: {
3637 // uadd.sat(a, b) == 0 -> (a | b) == 0
3638 // umax(a, b) == 0 -> (a | b) == 0
3639 if (C.isZero() && II->hasOneUse()) {
3641 return new ICmpInst(Pred, Or, Constant::getNullValue(Ty));
3642 }
3643 break;
3644 }
3645
3646 case Intrinsic::ssub_sat:
3647 // ssub.sat(a, b) == 0 -> a == b
3648 if (C.isZero())
3649 return new ICmpInst(Pred, II->getArgOperand(0), II->getArgOperand(1));
3650 break;
3651 case Intrinsic::usub_sat: {
3652 // usub.sat(a, b) == 0 -> a <= b
3653 if (C.isZero()) {
3654 ICmpInst::Predicate NewPred =
3656 return new ICmpInst(NewPred, II->getArgOperand(0), II->getArgOperand(1));
3657 }
3658 break;
3659 }
3660 default:
3661 break;
3662 }
3663
3664 return nullptr;
3665}
3666
3667/// Fold an icmp with LLVM intrinsics
3668static Instruction *
3670 InstCombiner::BuilderTy &Builder) {
3671 assert(Cmp.isEquality());
3672
3673 ICmpInst::Predicate Pred = Cmp.getPredicate();
3674 Value *Op0 = Cmp.getOperand(0);
3675 Value *Op1 = Cmp.getOperand(1);
3676 const auto *IIOp0 = dyn_cast<IntrinsicInst>(Op0);
3677 const auto *IIOp1 = dyn_cast<IntrinsicInst>(Op1);
3678 if (!IIOp0 || !IIOp1 || IIOp0->getIntrinsicID() != IIOp1->getIntrinsicID())
3679 return nullptr;
3680
3681 switch (IIOp0->getIntrinsicID()) {
3682 case Intrinsic::bswap:
3683 case Intrinsic::bitreverse:
3684 // If both operands are byte-swapped or bit-reversed, just compare the
3685 // original values.
3686 return new ICmpInst(Pred, IIOp0->getOperand(0), IIOp1->getOperand(0));
3687 case Intrinsic::fshl:
3688 case Intrinsic::fshr: {
3689 // If both operands are rotated by same amount, just compare the
3690 // original values.
3691 if (IIOp0->getOperand(0) != IIOp0->getOperand(1))
3692 break;
3693 if (IIOp1->getOperand(0) != IIOp1->getOperand(1))
3694 break;
3695 if (IIOp0->getOperand(2) == IIOp1->getOperand(2))
3696 return new ICmpInst(Pred, IIOp0->getOperand(0), IIOp1->getOperand(0));
3697
3698 // rotate(X, AmtX) == rotate(Y, AmtY)
3699 // -> rotate(X, AmtX - AmtY) == Y
3700 // Do this if either both rotates have one use or if only one has one use
3701 // and AmtX/AmtY are constants.
3702 unsigned OneUses = IIOp0->hasOneUse() + IIOp1->hasOneUse();
3703 if (OneUses == 2 ||
3704 (OneUses == 1 && match(IIOp0->getOperand(2), m_ImmConstant()) &&
3705 match(IIOp1->getOperand(2), m_ImmConstant()))) {
3706 Value *SubAmt =
3707 Builder.CreateSub(IIOp0->getOperand(2), IIOp1->getOperand(2));
3708 Value *CombinedRotate = Builder.CreateIntrinsic(
3709 Op0->getType(), IIOp0->getIntrinsicID(),
3710 {IIOp0->getOperand(0), IIOp0->getOperand(0), SubAmt});
3711 return new ICmpInst(Pred, IIOp1->getOperand(0), CombinedRotate);
3712 }
3713 } break;
3714 default:
3715 break;
3716 }
3717
3718 return nullptr;
3719}
3720
3721/// Try to fold integer comparisons with a constant operand: icmp Pred X, C
3722/// where X is some kind of instruction and C is AllowUndef.
3723/// TODO: Move more folds which allow undef to this function.
3726 const APInt &C) {
3727 const ICmpInst::Predicate Pred = Cmp.getPredicate();
3728 if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0))) {
3729 switch (II->getIntrinsicID()) {
3730 default:
3731 break;
3732 case Intrinsic::fshl:
3733 case Intrinsic::fshr:
3734 if (Cmp.isEquality() && II->getArgOperand(0) == II->getArgOperand(1)) {
3735 // (rot X, ?) == 0/-1 --> X == 0/-1
3736 if (C.isZero() || C.isAllOnes())
3737 return new ICmpInst(Pred, II->getArgOperand(0), Cmp.getOperand(1));
3738 }
3739 break;
3740 }
3741 }
3742
3743 return nullptr;
3744}
3745
3746/// Fold an icmp with BinaryOp and constant operand: icmp Pred BO, C.
3748 BinaryOperator *BO,
3749 const APInt &C) {
3750 switch (BO->getOpcode()) {
3751 case Instruction::Xor:
3752 if (Instruction *I = foldICmpXorConstant(Cmp, BO, C))
3753 return I;
3754 break;
3755 case Instruction::And:
3756 if (Instruction *I = foldICmpAndConstant(Cmp, BO, C))
3757 return I;
3758 break;
3759 case Instruction::Or:
3760 if (Instruction *I = foldICmpOrConstant(Cmp, BO, C))
3761 return I;
3762 break;
3763 case Instruction::Mul:
3764 if (Instruction *I = foldICmpMulConstant(Cmp, BO, C))
3765 return I;
3766 break;
3767 case Instruction::Shl:
3768 if (Instruction *I = foldICmpShlConstant(Cmp, BO, C))
3769 return I;
3770 break;
3771 case Instruction::LShr:
3772 case Instruction::AShr:
3773 if (Instruction *I = foldICmpShrConstant(Cmp, BO, C))
3774 return I;
3775 break;
3776 case Instruction::SRem:
3777 if (Instruction *I = foldICmpSRemConstant(Cmp, BO, C))
3778 return I;
3779 break;
3780 case Instruction::UDiv:
3781 if (Instruction *I = foldICmpUDivConstant(Cmp, BO, C))
3782 return I;
3783 [[fallthrough]];
3784 case Instruction::SDiv:
3785 if (Instruction *I = foldICmpDivConstant(Cmp, BO, C))
3786 return I;
3787 break;
3788 case Instruction::Sub:
3789 if (Instruction *I = foldICmpSubConstant(Cmp, BO, C))
3790 return I;
3791 break;
3792 case Instruction::Add:
3793 if (Instruction *I = foldICmpAddConstant(Cmp, BO, C))
3794 return I;
3795 break;
3796 default:
3797 break;
3798 }
3799
3800 // TODO: These folds could be refactored to be part of the above calls.
3801 return foldICmpBinOpEqualityWithConstant(Cmp, BO, C);
3802}
3803
3804static Instruction *
3806 SaturatingInst *II, const APInt &C,
3807 InstCombiner::BuilderTy &Builder) {
3808 // This transform may end up producing more than one instruction for the
3809 // intrinsic, so limit it to one user of the intrinsic.
3810 if (!II->hasOneUse())
3811 return nullptr;
3812
3813 // Let Y = [add/sub]_sat(X, C) pred C2
3814 // SatVal = The saturating value for the operation
3815 // WillWrap = Whether or not the operation will underflow / overflow
3816 // => Y = (WillWrap ? SatVal : (X binop C)) pred C2
3817 // => Y = WillWrap ? (SatVal pred C2) : ((X binop C) pred C2)
3818 //
3819 // When (SatVal pred C2) is true, then
3820 // Y = WillWrap ? true : ((X binop C) pred C2)
3821 // => Y = WillWrap || ((X binop C) pred C2)
3822 // else
3823 // Y = WillWrap ? false : ((X binop C) pred C2)
3824 // => Y = !WillWrap ? ((X binop C) pred C2) : false
3825 // => Y = !WillWrap && ((X binop C) pred C2)
3826 Value *Op0 = II->getOperand(0);
3827 Value *Op1 = II->getOperand(1);
3828
3829 const APInt *COp1;
3830 // This transform only works when the intrinsic has an integral constant or
3831 // splat vector as the second operand.
3832 if (!match(Op1, m_APInt(COp1)))
3833 return nullptr;
3834
3835 APInt SatVal;
3836 switch (II->getIntrinsicID()) {
3837 default:
3839 "This function only works with usub_sat and uadd_sat for now!");
3840 case Intrinsic::uadd_sat:
3841 SatVal = APInt::getAllOnes(C.getBitWidth());
3842 break;
3843 case Intrinsic::usub_sat:
3844 SatVal = APInt::getZero(C.getBitWidth());
3845 break;
3846 }
3847
3848 // Check (SatVal pred C2)
3849 bool SatValCheck = ICmpInst::compare(SatVal, C, Pred);
3850
3851 // !WillWrap.
3853 II->getBinaryOp(), *COp1, II->getNoWrapKind());
3854
3855 // WillWrap.
3856 if (SatValCheck)
3857 C1 = C1.inverse();
3858
3860 if (II->getBinaryOp() == Instruction::Add)
3861 C2 = C2.sub(*COp1);
3862 else
3863 C2 = C2.add(*COp1);
3864
3865 Instruction::BinaryOps CombiningOp =
3866 SatValCheck ? Instruction::BinaryOps::Or : Instruction::BinaryOps::And;
3867
3868 std::optional<ConstantRange> Combination;
3869 if (CombiningOp == Instruction::BinaryOps::Or)
3870 Combination = C1.exactUnionWith(C2);
3871 else /* CombiningOp == Instruction::BinaryOps::And */
3872 Combination = C1.exactIntersectWith(C2);
3873
3874 if (!Combination)
3875 return nullptr;
3876
3877 CmpInst::Predicate EquivPred;
3878 APInt EquivInt;
3879 APInt EquivOffset;
3880
3881 Combination->getEquivalentICmp(EquivPred, EquivInt, EquivOffset);
3882
3883 return new ICmpInst(
3884 EquivPred,
3885 Builder.CreateAdd(Op0, ConstantInt::get(Op1->getType(), EquivOffset)),
3886 ConstantInt::get(Op1->getType(), EquivInt));
3887}
3888
3889/// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
3891 IntrinsicInst *II,
3892 const APInt &C) {
3893 ICmpInst::Predicate Pred = Cmp.getPredicate();
3894
3895 // Handle folds that apply for any kind of icmp.
3896 switch (II->getIntrinsicID()) {
3897 default:
3898 break;
3899 case Intrinsic::uadd_sat:
3900 case Intrinsic::usub_sat:
3901 if (auto *Folded = foldICmpUSubSatOrUAddSatWithConstant(
3902 Pred, cast<SaturatingInst>(II), C, Builder))
3903 return Folded;
3904 break;
3905 case Intrinsic::ctpop: {
3906 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
3907 if (Instruction *R = foldCtpopPow2Test(Cmp, II, C, Builder, Q))
3908 return R;
3909 } break;
3910 }
3911
3912 if (Cmp.isEquality())
3913 return foldICmpEqIntrinsicWithConstant(Cmp, II, C);
3914
3915 Type *Ty = II->getType();
3916 unsigned BitWidth = C.getBitWidth();
3917 switch (II->getIntrinsicID()) {
3918 case Intrinsic::ctpop: {
3919 // (ctpop X > BitWidth - 1) --> X == -1
3920 Value *X = II->getArgOperand(0);
3921 if (C == BitWidth - 1 && Pred == ICmpInst::ICMP_UGT)
3922 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, X,
3924 // (ctpop X < BitWidth) --> X != -1
3925 if (C == BitWidth && Pred == ICmpInst::ICMP_ULT)
3926 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, X,
3928 break;
3929 }
3930 case Intrinsic::ctlz: {
3931 // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000
3932 if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) {
3933 unsigned Num = C.getLimitedValue();
3934 APInt Limit = APInt::getOneBitSet(BitWidth, BitWidth - Num - 1);
3935 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT,
3936 II->getArgOperand(0), ConstantInt::get(Ty, Limit));
3937 }
3938
3939 // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111
3940 if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) {
3941 unsigned Num = C.getLimitedValue();
3943 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT,
3944 II->getArgOperand(0), ConstantInt::get(Ty, Limit));
3945 }
3946 break;
3947 }
3948 case Intrinsic::cttz: {
3949 // Limit to one use to ensure we don't increase instruction count.
3950 if (!II->hasOneUse())
3951 return nullptr;
3952
3953 // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0
3954 if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) {
3955 APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue() + 1);
3956 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ,
3957 Builder.CreateAnd(II->getArgOperand(0), Mask),
3959 }
3960
3961 // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0
3962 if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) {
3963 APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue());
3964 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE,
3965 Builder.CreateAnd(II->getArgOperand(0), Mask),
3967 }
3968 break;
3969 }
3970 case Intrinsic::ssub_sat:
3971 // ssub.sat(a, b) spred 0 -> a spred b
3972 if (ICmpInst::isSigned(Pred)) {
3973 if (C.isZero())
3974 return new ICmpInst(Pred, II->getArgOperand(0), II->getArgOperand(1));
3975 // X s<= 0 is cannonicalized to X s< 1
3976 if (Pred == ICmpInst::ICMP_SLT && C.isOne())
3977 return new ICmpInst(ICmpInst::ICMP_SLE, II->getArgOperand(0),
3978 II->getArgOperand(1));
3979 // X s>= 0 is cannonicalized to X s> -1
3980 if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
3981 return new ICmpInst(ICmpInst::ICMP_SGE, II->getArgOperand(0),
3982 II->getArgOperand(1));
3983 }
3984 break;
3985 default:
3986 break;
3987 }
3988
3989 return nullptr;
3990}
3991
3992/// Handle icmp with constant (but not simple integer constant) RHS.
3994 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
3995 Constant *RHSC = dyn_cast<Constant>(Op1);
3996 Instruction *LHSI = dyn_cast<Instruction>(Op0);
3997 if (!RHSC || !LHSI)
3998 return nullptr;
3999
4000 switch (LHSI->getOpcode()) {
4001 case Instruction::PHI:
4002 if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI)))
4003 return NV;
4004 break;
4005 case Instruction::IntToPtr:
4006 // icmp pred inttoptr(X), null -> icmp pred X, 0
4007 if (RHSC->isNullValue() &&
4008 DL.getIntPtrType(RHSC->getType()) == LHSI->getOperand(0)->getType())
4009 return new ICmpInst(
4010 I.getPredicate(), LHSI->getOperand(0),
4012 break;
4013
4014 case Instruction::Load:
4015 // Try to optimize things like "A[i] > 4" to index computations.
4016 if (GetElementPtrInst *GEP =
4017 dyn_cast<GetElementPtrInst>(LHSI->getOperand(0)))
4018 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)))
4019 if (Instruction *Res =
4020 foldCmpLoadFromIndexedGlobal(cast<LoadInst>(LHSI), GEP, GV, I))
4021 return Res;
4022 break;
4023 }
4024
4025 return nullptr;
4026}
4027
4029 SelectInst *SI, Value *RHS,
4030 const ICmpInst &I) {
4031 // Try to fold the comparison into the select arms, which will cause the
4032 // select to be converted into a logical and/or.
4033 auto SimplifyOp = [&](Value *Op, bool SelectCondIsTrue) -> Value * {
4034 if (Value *Res = simplifyICmpInst(Pred, Op, RHS, SQ))
4035 return Res;
4036 if (std::optional<bool> Impl = isImpliedCondition(
4037 SI->getCondition(), Pred, Op, RHS, DL, SelectCondIsTrue))
4038 return ConstantInt::get(I.getType(), *Impl);
4039 return nullptr;
4040 };
4041
4042 ConstantInt *CI = nullptr;
4043 Value *Op1 = SimplifyOp(SI->getOperand(1), true);
4044 if (Op1)
4045 CI = dyn_cast<ConstantInt>(Op1);
4046
4047 Value *Op2 = SimplifyOp(SI->getOperand(2), false);
4048 if (Op2)
4049 CI = dyn_cast<ConstantInt>(Op2);
4050
4051 // We only want to perform this transformation if it will not lead to
4052 // additional code. This is true if either both sides of the select
4053 // fold to a constant (in which case the icmp is replaced with a select
4054 // which will usually simplify) or this is the only user of the
4055 // select (in which case we are trading a select+icmp for a simpler
4056 // select+icmp) or all uses of the select can be replaced based on
4057 // dominance information ("Global cases").
4058 bool Transform = false;
4059 if (Op1 && Op2)
4060 Transform = true;
4061 else if (Op1 || Op2) {
4062 // Local case
4063 if (SI->hasOneUse())
4064 Transform = true;
4065 // Global cases
4066 else if (CI && !CI->isZero())
4067 // When Op1 is constant try replacing select with second operand.
4068 // Otherwise Op2 is constant and try replacing select with first
4069 // operand.
4070 Transform = replacedSelectWithOperand(SI, &I, Op1 ? 2 : 1);
4071 }
4072 if (Transform) {
4073 if (!Op1)
4074 Op1 = Builder.CreateICmp(Pred, SI->getOperand(1), RHS, I.getName());
4075 if (!Op2)
4076 Op2 = Builder.CreateICmp(Pred, SI->getOperand(2), RHS, I.getName());
4077 return SelectInst::Create(SI->getOperand(0), Op1, Op2);
4078 }
4079
4080 return nullptr;
4081}
4082
4083// Returns whether V is a Mask ((X + 1) & X == 0) or ~Mask (-Pow2OrZero)
4084static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q,
4085 unsigned Depth = 0) {
4086 if (Not ? match(V, m_NegatedPower2OrZero()) : match(V, m_LowBitMaskOrZero()))
4087 return true;
4088 if (V->getType()->getScalarSizeInBits() == 1)
4089 return true;
4091 return false;
4092 Value *X;
4093 const Instruction *I = dyn_cast<Instruction>(V);
4094 if (!I)
4095 return false;
4096 switch (I->getOpcode()) {
4097 case Instruction::ZExt:
4098 // ZExt(Mask) is a Mask.
4099 return !Not && isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4100 case Instruction::SExt:
4101 // SExt(Mask) is a Mask.
4102 // SExt(~Mask) is a ~Mask.
4103 return isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4104 case Instruction::And:
4105 case Instruction::Or:
4106 // Mask0 | Mask1 is a Mask.
4107 // Mask0 & Mask1 is a Mask.
4108 // ~Mask0 | ~Mask1 is a ~Mask.
4109 // ~Mask0 & ~Mask1 is a ~Mask.
4110 return isMaskOrZero(I->getOperand(1), Not, Q, Depth) &&
4111 isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4112 case Instruction::Xor:
4113 if (match(V, m_Not(m_Value(X))))
4114 return isMaskOrZero(X, !Not, Q, Depth);
4115
4116 // (X ^ -X) is a ~Mask
4117 if (Not)
4118 return match(V, m_c_Xor(m_Value(X), m_Neg(m_Deferred(X))));
4119 // (X ^ (X - 1)) is a Mask
4120 else
4121 return match(V, m_c_Xor(m_Value(X), m_Add(m_Deferred(X), m_AllOnes())));
4122 case Instruction::Select:
4123 // c ? Mask0 : Mask1 is a Mask.
4124 return isMaskOrZero(I->getOperand(1), Not, Q, Depth) &&
4125 isMaskOrZero(I->getOperand(2), Not, Q, Depth);
4126 case Instruction::Shl:
4127 // (~Mask) << X is a ~Mask.
4128 return Not && isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4129 case Instruction::LShr:
4130 // Mask >> X is a Mask.
4131 return !Not && isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4132 case Instruction::AShr:
4133 // Mask s>> X is a Mask.
4134 // ~Mask s>> X is a ~Mask.
4135 return isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4136 case Instruction::Add:
4137 // Pow2 - 1 is a Mask.
4138 if (!Not && match(I->getOperand(1), m_AllOnes()))
4139 return isKnownToBeAPowerOfTwo(I->getOperand(0), Q.DL, /*OrZero*/ true,
4140 Depth, Q.AC, Q.CxtI, Q.DT);
4141 break;
4142 case Instruction::Sub:
4143 // -Pow2 is a ~Mask.
4144 if (Not && match(I->getOperand(0), m_Zero()))
4145 return isKnownToBeAPowerOfTwo(I->getOperand(1), Q.DL, /*OrZero*/ true,
4146 Depth, Q.AC, Q.CxtI, Q.DT);
4147 break;
4148 case Instruction::Call: {
4149 if (auto *II = dyn_cast<IntrinsicInst>(I)) {
4150 switch (II->getIntrinsicID()) {
4151 // min/max(Mask0, Mask1) is a Mask.
4152 // min/max(~Mask0, ~Mask1) is a ~Mask.
4153 case Intrinsic::umax:
4154 case Intrinsic::smax:
4155 case Intrinsic::umin:
4156 case Intrinsic::smin:
4157 return isMaskOrZero(II->getArgOperand(1), Not, Q, Depth) &&
4158 isMaskOrZero(II->getArgOperand(0), Not, Q, Depth);
4159
4160 // In the context of masks, bitreverse(Mask) == ~Mask
4161 case Intrinsic::bitreverse:
4162 return isMaskOrZero(II->getArgOperand(0), !Not, Q, Depth);
4163 default:
4164 break;
4165 }
4166 }
4167 break;
4168 }
4169 default:
4170 break;
4171 }
4172 return false;
4173}
4174
4175/// Some comparisons can be simplified.
4176/// In this case, we are looking for comparisons that look like
4177/// a check for a lossy truncation.
4178/// Folds:
4179/// icmp SrcPred (x & Mask), x to icmp DstPred x, Mask
4180/// icmp eq/ne (x & ~Mask), 0 to icmp DstPred x, Mask
4181/// Where Mask is some pattern that produces all-ones in low bits:
4182/// (-1 >> y)
4183/// ((-1 << y) >> y) <- non-canonical, has extra uses
4184/// ~(-1 << y)
4185/// ((1 << y) + (-1)) <- non-canonical, has extra uses
4186/// The Mask can be a constant, too.
4187/// For some predicates, the operands are commutative.
4188/// For others, x can only be on a specific side.
4190 Value *Op1, const SimplifyQuery &Q,
4191 InstCombiner &IC) {
4192 Value *X, *M;
4193 bool NeedsNot = false;
4194
4195 auto CheckMask = [&](Value *V, bool Not) {
4196 if (ICmpInst::isSigned(Pred) && !match(V, m_ImmConstant()))
4197 return false;
4198 return isMaskOrZero(V, Not, Q);
4199 };
4200
4201 if (match(Op0, m_c_And(m_Specific(Op1), m_Value(M))) &&
4202 CheckMask(M, /*Not*/ false)) {
4203 X = Op1;
4204 } else if (match(Op1, m_Zero()) && ICmpInst::isEquality(Pred) &&
4205 match(Op0, m_OneUse(m_And(m_Value(X), m_Value(M))))) {
4206 NeedsNot = true;
4207 if (IC.isFreeToInvert(X, X->hasOneUse()) && CheckMask(X, /*Not*/ true))
4208 std::swap(X, M);
4209 else if (!IC.isFreeToInvert(M, M->hasOneUse()) ||
4210 !CheckMask(M, /*Not*/ true))
4211 return nullptr;
4212 } else {
4213 return nullptr;
4214 }
4215
4216 ICmpInst::Predicate DstPred;
4217 switch (Pred) {
4219 // x & (-1 >> y) == x -> x u<= (-1 >> y)
4221 break;
4223 // x & (-1 >> y) != x -> x u> (-1 >> y)
4225 break;
4227 // x & (-1 >> y) u< x -> x u> (-1 >> y)
4228 // x u> x & (-1 >> y) -> x u> (-1 >> y)
4230 break;
4232 // x & (-1 >> y) u>= x -> x u<= (-1 >> y)
4233 // x u<= x & (-1 >> y) -> x u<= (-1 >> y)
4235 break;
4237 // x & (-1 >> y) s< x -> x s> (-1 >> y)
4238 // x s> x & (-1 >> y) -> x s> (-1 >> y)
4239 if (!match(M, m_Constant())) // Can not do this fold with non-constant.
4240 return nullptr;
4241 if (!match(M, m_NonNegative())) // Must not have any -1 vector elements.
4242 return nullptr;
4244 break;
4246 // x & (-1 >> y) s>= x -> x s<= (-1 >> y)
4247 // x s<= x & (-1 >> y) -> x s<= (-1 >> y)
4248 if (!match(M, m_Constant())) // Can not do this fold with non-constant.
4249 return nullptr;
4250 if (!match(M, m_NonNegative())) // Must not have any -1 vector elements.
4251 return nullptr;
4253 break;
4256 return nullptr;
4259 llvm_unreachable("Instsimplify took care of commut. variant");
4260 break;
4261 default:
4262 llvm_unreachable("All possible folds are handled.");
4263 }
4264
4265 // The mask value may be a vector constant that has undefined elements. But it
4266 // may not be safe to propagate those undefs into the new compare, so replace
4267 // those elements by copying an existing, defined, and safe scalar constant.
4268 Type *OpTy = M->getType();
4269 auto *VecC = dyn_cast<Constant>(M);
4270 auto *OpVTy = dyn_cast<FixedVectorType>(OpTy);
4271 if (OpVTy && VecC && VecC->containsUndefOrPoisonElement()) {
4272 Constant *SafeReplacementConstant = nullptr;
4273 for (unsigned i = 0, e = OpVTy->getNumElements(); i != e; ++i) {
4274 if (!isa<UndefValue>(VecC->getAggregateElement(i))) {
4275 SafeReplacementConstant = VecC->getAggregateElement(i);
4276 break;
4277 }
4278 }
4279 assert(SafeReplacementConstant && "Failed to find undef replacement");
4280 M = Constant::replaceUndefsWith(VecC, SafeReplacementConstant);
4281 }
4282
4283 if (NeedsNot)
4284 M = IC.Builder.CreateNot(M);
4285 return IC.Builder.CreateICmp(DstPred, X, M);
4286}
4287
4288/// Some comparisons can be simplified.
4289/// In this case, we are looking for comparisons that look like
4290/// a check for a lossy signed truncation.
4291/// Folds: (MaskedBits is a constant.)
4292/// ((%x << MaskedBits) a>> MaskedBits) SrcPred %x
4293/// Into:
4294/// (add %x, (1 << (KeptBits-1))) DstPred (1 << KeptBits)
4295/// Where KeptBits = bitwidth(%x) - MaskedBits
4296static Value *
4298 InstCombiner::BuilderTy &Builder) {
4299 ICmpInst::Predicate SrcPred;
4300 Value *X;
4301 const APInt *C0, *C1; // FIXME: non-splats, potentially with undef.
4302 // We are ok with 'shl' having multiple uses, but 'ashr' must be one-use.
4303 if (!match(&I, m_c_ICmp(SrcPred,
4305 m_APInt(C1))),
4306 m_Deferred(X))))
4307 return nullptr;
4308
4309 // Potential handling of non-splats: for each element:
4310 // * if both are undef, replace with constant 0.
4311 // Because (1<<0) is OK and is 1, and ((1<<0)>>1) is also OK and is 0.
4312 // * if both are not undef, and are different, bailout.
4313 // * else, only one is undef, then pick the non-undef one.
4314
4315 // The shift amount must be equal.
4316 if (*C0 != *C1)
4317 return nullptr;
4318 const APInt &MaskedBits = *C0;
4319 assert(MaskedBits != 0 && "shift by zero should be folded away already.");
4320
4321 ICmpInst::Predicate DstPred;
4322 switch (SrcPred) {
4324 // ((%x << MaskedBits) a>> MaskedBits) == %x
4325 // =>
4326 // (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits)
4328 break;
4330 // ((%x << MaskedBits) a>> MaskedBits) != %x
4331 // =>
4332 // (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits)
4334 break;
4335 // FIXME: are more folds possible?
4336 default:
4337 return nullptr;
4338 }
4339
4340 auto *XType = X->getType();
4341 const unsigned XBitWidth = XType->getScalarSizeInBits();
4342 const APInt BitWidth = APInt(XBitWidth, XBitWidth);
4343 assert(BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched");
4344
4345 // KeptBits = bitwidth(%x) - MaskedBits
4346 const APInt KeptBits = BitWidth - MaskedBits;
4347 assert(KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable");
4348 // ICmpCst = (1 << KeptBits)
4349 const APInt ICmpCst = APInt(XBitWidth, 1).shl(KeptBits);
4350 assert(ICmpCst.isPowerOf2());
4351 // AddCst = (1 << (KeptBits-1))
4352 const APInt AddCst = ICmpCst.lshr(1);
4353 assert(AddCst.ult(ICmpCst) && AddCst.isPowerOf2());
4354
4355 // T0 = add %x, AddCst
4356 Value *T0 = Builder.CreateAdd(X, ConstantInt::get(XType, AddCst));
4357 // T1 = T0 DstPred ICmpCst
4358 Value *T1 = Builder.CreateICmp(DstPred, T0, ConstantInt::get(XType, ICmpCst));
4359
4360 return T1;
4361}
4362
4363// Given pattern:
4364// icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
4365// we should move shifts to the same hand of 'and', i.e. rewrite as
4366// icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x)
4367// We are only interested in opposite logical shifts here.
4368// One of the shifts can be truncated.
4369// If we can, we want to end up creating 'lshr' shift.
4370static Value *
4372 InstCombiner::BuilderTy &Builder) {
4373 if (!I.isEquality() || !match(I.getOperand(1), m_Zero()) ||
4374 !I.getOperand(0)->hasOneUse())
4375 return nullptr;
4376
4377 auto m_AnyLogicalShift = m_LogicalShift(m_Value(), m_Value());
4378
4379 // Look for an 'and' of two logical shifts, one of which may be truncated.
4380 // We use m_TruncOrSelf() on the RHS to correctly handle commutative case.
4381 Instruction *XShift, *MaybeTruncation, *YShift;
4382 if (!match(
4383 I.getOperand(0),
4384 m_c_And(m_CombineAnd(m_AnyLogicalShift, m_Instruction(XShift)),
4386 m_AnyLogicalShift, m_Instruction(YShift))),
4387 m_Instruction(MaybeTruncation)))))
4388 return nullptr;
4389
4390 // We potentially looked past 'trunc', but only when matching YShift,
4391 // therefore YShift must have the widest type.
4392 Instruction *WidestShift = YShift;
4393 // Therefore XShift must have the shallowest type.
4394 // Or they both have identical types if there was no truncation.
4395 Instruction *NarrowestShift = XShift;
4396
4397 Type *WidestTy = WidestShift->getType();
4398 Type *NarrowestTy = NarrowestShift->getType();
4399 assert(NarrowestTy == I.getOperand(0)->getType() &&
4400 "We did not look past any shifts while matching XShift though.");
4401 bool HadTrunc = WidestTy != I.getOperand(0)->getType();
4402
4403 // If YShift is a 'lshr', swap the shifts around.
4404 if (match(YShift, m_LShr(m_Value(), m_Value())))
4405 std::swap(XShift, YShift);
4406
4407 // The shifts must be in opposite directions.
4408 auto XShiftOpcode = XShift->getOpcode();
4409 if (XShiftOpcode == YShift->getOpcode())
4410 return nullptr; // Do not care about same-direction shifts here.
4411
4412 Value *X, *XShAmt, *Y, *YShAmt;
4413 match(XShift, m_BinOp(m_Value(X), m_ZExtOrSelf(m_Value(XShAmt))));
4414 match(YShift, m_BinOp(m_Value(Y), m_ZExtOrSelf(m_Value(YShAmt))));
4415
4416 // If one of the values being shifted is a constant, then we will end with
4417 // and+icmp, and [zext+]shift instrs will be constant-folded. If they are not,
4418 // however, we will need to ensure that we won't increase instruction count.
4419 if (!isa<Constant>(X) && !isa<Constant>(Y)) {
4420 // At least one of the hands of the 'and' should be one-use shift.
4421 if (!match(I.getOperand(0),
4422 m_c_And(m_OneUse(m_AnyLogicalShift), m_Value())))
4423 return nullptr;
4424 if (HadTrunc) {
4425 // Due to the 'trunc', we will need to widen X. For that either the old
4426 // 'trunc' or the shift amt in the non-truncated shift should be one-use.
4427 if (!MaybeTruncation->hasOneUse() &&
4428 !NarrowestShift->getOperand(1)->hasOneUse())
4429 return nullptr;
4430 }
4431 }
4432
4433 // We have two shift amounts from two different shifts. The types of those
4434 // shift amounts may not match. If that's the case let's bailout now.
4435 if (XShAmt->getType() != YShAmt->getType())
4436 return nullptr;
4437
4438 // As input, we have the following pattern:
4439 // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
4440 // We want to rewrite that as:
4441 // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x)
4442 // While we know that originally (Q+K) would not overflow
4443 // (because 2 * (N-1) u<= iN -1), we have looked past extensions of
4444 // shift amounts. so it may now overflow in smaller bitwidth.
4445 // To ensure that does not happen, we need to ensure that the total maximal
4446 // shift amount is still representable in that smaller bit width.
4447 unsigned MaximalPossibleTotalShiftAmount =
4448 (WidestTy->getScalarSizeInBits() - 1) +
4449 (NarrowestTy->getScalarSizeInBits() - 1);
4450 APInt MaximalRepresentableShiftAmount =
4452 if (MaximalRepresentableShiftAmount.ult(MaximalPossibleTotalShiftAmount))
4453 return nullptr;
4454
4455 // Can we fold (XShAmt+YShAmt) ?
4456 auto *NewShAmt = dyn_cast_or_null<Constant>(
4457 simplifyAddInst(XShAmt, YShAmt, /*isNSW=*/false,
4458 /*isNUW=*/false, SQ.getWithInstruction(&I)));
4459 if (!NewShAmt)
4460 return nullptr;
4461 if (NewShAmt->getType() != WidestTy) {
4462 NewShAmt =
4463 ConstantFoldCastOperand(Instruction::ZExt, NewShAmt, WidestTy, SQ.DL);
4464 if (!NewShAmt)
4465 return nullptr;
4466 }
4467 unsigned WidestBitWidth = WidestTy->getScalarSizeInBits();
4468
4469 // Is the new shift amount smaller than the bit width?
4470 // FIXME: could also rely on ConstantRange.
4471 if (!match(NewShAmt,
4473 APInt(WidestBitWidth, WidestBitWidth))))
4474 return nullptr;
4475
4476 // An extra legality check is needed if we had trunc-of-lshr.
4477 if (HadTrunc && match(WidestShift, m_LShr(m_Value(), m_Value()))) {
4478 auto CanFold = [NewShAmt, WidestBitWidth, NarrowestShift, SQ,
4479 WidestShift]() {
4480 // It isn't obvious whether it's worth it to analyze non-constants here.
4481 // Also, let's basically give up on non-splat cases, pessimizing vectors.
4482 // If *any* of these preconditions matches we can perform the fold.
4483 Constant *NewShAmtSplat = NewShAmt->getType()->isVectorTy()
4484 ? NewShAmt->getSplatValue()
4485 : NewShAmt;
4486 // If it's edge-case shift (by 0 or by WidestBitWidth-1) we can fold.
4487 if (NewShAmtSplat &&
4488 (NewShAmtSplat->isNullValue() ||
4489 NewShAmtSplat->getUniqueInteger() == WidestBitWidth - 1))
4490 return true;
4491 // We consider *min* leading zeros so a single outlier
4492 // blocks the transform as opposed to allowing it.
4493 if (auto *C = dyn_cast<Constant>(NarrowestShift->getOperand(0))) {
4494 KnownBits Known = computeKnownBits(C, SQ.DL);
4495 unsigned MinLeadZero = Known.countMinLeadingZeros();
4496 // If the value being shifted has at most lowest bit set we can fold.
4497 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero;
4498 if (MaxActiveBits <= 1)
4499 return true;
4500 // Precondition: NewShAmt u<= countLeadingZeros(C)
4501 if (NewShAmtSplat && NewShAmtSplat->getUniqueInteger().ule(MinLeadZero))
4502 return true;
4503 }
4504 if (auto *C = dyn_cast<Constant>(WidestShift->getOperand(0))) {
4505 KnownBits Known = computeKnownBits(C, SQ.DL);
4506 unsigned MinLeadZero = Known.countMinLeadingZeros();
4507 // If the value being shifted has at most lowest bit set we can fold.
4508 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero;
4509 if (MaxActiveBits <= 1)
4510 return true;
4511 // Precondition: ((WidestBitWidth-1)-NewShAmt) u<= countLeadingZeros(C)
4512 if (NewShAmtSplat) {
4513 APInt AdjNewShAmt =
4514 (WidestBitWidth - 1) - NewShAmtSplat->getUniqueInteger();
4515 if (AdjNewShAmt.ule(MinLeadZero))
4516 return true;
4517 }
4518 }
4519 return false; // Can't tell if it's ok.
4520 };
4521 if (!CanFold())
4522 return nullptr;
4523 }
4524
4525 // All good, we can do this fold.
4526 X = Builder.CreateZExt(X, WidestTy);
4527 Y = Builder.CreateZExt(Y, WidestTy);
4528 // The shift is the same that was for X.
4529 Value *T0 = XShiftOpcode == Instruction::BinaryOps::LShr
4530 ? Builder.CreateLShr(X, NewShAmt)
4531 : Builder.CreateShl(X, NewShAmt);
4532 Value *T1 = Builder.CreateAnd(T0, Y);
4533 return Builder.CreateICmp(I.getPredicate(), T1,
4534 Constant::getNullValue(WidestTy));
4535}
4536
4537/// Fold
4538/// (-1 u/ x) u< y
4539/// ((x * y) ?/ x) != y
4540/// to
4541/// @llvm.?mul.with.overflow(x, y) plus extraction of overflow bit
4542/// Note that the comparison is commutative, while inverted (u>=, ==) predicate
4543/// will mean that we are looking for the opposite answer.
4546 Value *X, *Y;
4548 Instruction *Div;
4549 bool NeedNegation;
4550 // Look for: (-1 u/ x) u</u>= y
4551 if (!I.isEquality() &&
4552 match(&I, m_c_ICmp(Pred,
4554 m_Instruction(Div)),
4555 m_Value(Y)))) {
4556 Mul = nullptr;
4557
4558 // Are we checking that overflow does not happen, or does happen?
4559 switch (Pred) {
4561 NeedNegation = false;
4562 break; // OK
4564 NeedNegation = true;
4565 break; // OK
4566 default:
4567 return nullptr; // Wrong predicate.
4568 }
4569 } else // Look for: ((x * y) / x) !=/== y
4570 if (I.isEquality() &&
4571 match(&I,
4572 m_c_ICmp(Pred, m_Value(Y),
4575 m_Value(X)),
4577 m_Deferred(X))),
4578 m_Instruction(Div))))) {
4579 NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ;
4580 } else
4581 return nullptr;
4582
4584 // If the pattern included (x * y), we'll want to insert new instructions
4585 // right before that original multiplication so that we can replace it.
4586 bool MulHadOtherUses = Mul && !Mul->hasOneUse();
4587 if (MulHadOtherUses)
4589
4590 Function *F = Intrinsic::getDeclaration(I.getModule(),
4591 Div->getOpcode() == Instruction::UDiv
4592 ? Intrinsic::umul_with_overflow
4593 : Intrinsic::smul_with_overflow,
4594 X->getType());
4595 CallInst *Call = Builder.CreateCall(F, {X, Y}, "mul");
4596
4597 // If the multiplication was used elsewhere, to ensure that we don't leave
4598 // "duplicate" instructions, replace uses of that original multiplication
4599 // with the multiplication result from the with.overflow intrinsic.
4600 if (MulHadOtherUses)
4601 replaceInstUsesWith(*Mul, Builder.CreateExtractValue(Call, 0, "mul.val"));
4602
4603 Value *Res = Builder.CreateExtractValue(Call, 1, "mul.ov");
4604 if (NeedNegation) // This technically increases instruction count.
4605 Res = Builder.CreateNot(Res, "mul.not.ov");
4606
4607 // If we replaced the mul, erase it. Do this after all uses of Builder,
4608 // as the mul is used as insertion point.
4609 if (MulHadOtherUses)
4611
4612 return Res;
4613}
4614
4616 InstCombiner::BuilderTy &Builder) {
4617 CmpInst::Predicate Pred;
4618 Value *X;
4619 if (match(&I, m_c_ICmp(Pred, m_NSWNeg(m_Value(X)), m_Deferred(X)))) {
4620
4621 if (ICmpInst::isSigned(Pred))
4622 Pred = ICmpInst::getSwappedPredicate(Pred);
4623 else if (ICmpInst::isUnsigned(Pred))
4624 Pred = ICmpInst::getSignedPredicate(Pred);
4625 // else for equality-comparisons just keep the predicate.
4626
4627 return ICmpInst::Create(Instruction::ICmp, Pred, X,
4628 Constant::getNullValue(X->getType()), I.getName());
4629 }
4630
4631 // A value is not equal to its negation unless that value is 0 or
4632 // MinSignedValue, ie: a != -a --> (a & MaxSignedVal) != 0
4633 if (match(&I, m_c_ICmp(Pred, m_OneUse(m_Neg(m_Value(X))), m_Deferred(X))) &&
4634 ICmpInst::isEquality(Pred)) {
4635 Type *Ty = X->getType();
4637 Constant *MaxSignedVal =
4638 ConstantInt::get(Ty, APInt::getSignedMaxValue(BitWidth));
4639 Value *And = Builder.CreateAnd(X, MaxSignedVal);
4640 Constant *Zero = Constant::getNullValue(Ty);
4641 return CmpInst::Create(Instruction::ICmp, Pred, And, Zero);
4642 }
4643
4644 return nullptr;
4645}
4646
4648 InstCombinerImpl &IC) {
4649 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *A;
4650 // Normalize and operand as operand 0.
4651 CmpInst::Predicate Pred = I.getPredicate();
4652 if (match(Op1, m_c_And(m_Specific(Op0), m_Value()))) {
4653 std::swap(Op0, Op1);
4654 Pred = ICmpInst::getSwappedPredicate(Pred);
4655 }
4656
4657 if (!match(Op0, m_c_And(m_Specific(Op1), m_Value(A))))
4658 return nullptr;
4659
4660 // (icmp (X & Y) u< X --> (X & Y) != X
4661 if (Pred == ICmpInst::ICMP_ULT)
4662 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
4663
4664 // (icmp (X & Y) u>= X --> (X & Y) == X
4665 if (Pred == ICmpInst::ICMP_UGE)
4666 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
4667
4668 return nullptr;
4669}
4670
4672 InstCombinerImpl &IC) {
4673 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *A;
4674
4675 // Normalize or operand as operand 0.
4676 CmpInst::Predicate Pred = I.getPredicate();
4677 if (match(Op1, m_c_Or(m_Specific(Op0), m_Value(A)))) {
4678 std::swap(Op0, Op1);
4679 Pred = ICmpInst::getSwappedPredicate(Pred);
4680 } else if (!match(Op0, m_c_Or(m_Specific(Op1), m_Value(A)))) {
4681 return nullptr;
4682 }
4683
4684 // icmp (X | Y) u<= X --> (X | Y) == X
4685 if (Pred == ICmpInst::ICMP_ULE)
4686 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
4687
4688 // icmp (X | Y) u> X --> (X | Y) != X
4689 if (Pred == ICmpInst::ICMP_UGT)
4690 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
4691
4692 if (ICmpInst::isEquality(Pred) && Op0->hasOneUse()) {
4693 // icmp (X | Y) eq/ne Y --> (X & ~Y) eq/ne 0 if Y is freely invertible
4694 if (Value *NotOp1 =
4695 IC.getFreelyInverted(Op1, Op1->hasOneUse(), &IC.Builder))
4696 return new ICmpInst(Pred, IC.Builder.CreateAnd(A, NotOp1),
4697 Constant::getNullValue(Op1->getType()));
4698 // icmp (X | Y) eq/ne Y --> (~X | Y) eq/ne -1 if X is freely invertible.
4699 if (Value *NotA = IC.getFreelyInverted(A, A->hasOneUse(), &IC.Builder))
4700 return new ICmpInst(Pred, IC.Builder.CreateOr(Op1, NotA),
4701 Constant::getAllOnesValue(Op1->getType()));
4702 }
4703 return nullptr;
4704}
4705
4707 InstCombinerImpl &IC) {
4708 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *A;
4709 // Normalize xor operand as operand 0.
4710 CmpInst::Predicate Pred = I.getPredicate();
4711 if (match(Op1, m_c_Xor(m_Specific(Op0), m_Value()))) {
4712 std::swap(Op0, Op1);
4713 Pred = ICmpInst::getSwappedPredicate(Pred);
4714 }
4715 if (!match(Op0, m_c_Xor(m_Specific(Op1), m_Value(A))))
4716 return nullptr;
4717
4718 // icmp (X ^ Y_NonZero) u>= X --> icmp (X ^ Y_NonZero) u> X
4719 // icmp (X ^ Y_NonZero) u<= X --> icmp (X ^ Y_NonZero) u< X
4720 // icmp (X ^ Y_NonZero) s>= X --> icmp (X ^ Y_NonZero) s> X
4721 // icmp (X ^ Y_NonZero) s<= X --> icmp (X ^ Y_NonZero) s< X
4723 if (PredOut != Pred &&
4724 isKnownNonZero(A, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
4725 return new ICmpInst(PredOut, Op0, Op1);
4726
4727 return nullptr;
4728}
4729
4730/// Try to fold icmp (binop), X or icmp X, (binop).
4731/// TODO: A large part of this logic is duplicated in InstSimplify's
4732/// simplifyICmpWithBinOp(). We should be able to share that and avoid the code
4733/// duplication.
4735 const SimplifyQuery &SQ) {
4737 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
4738
4739 // Special logic for binary operators.
4740 BinaryOperator *BO0 = dyn_cast<BinaryOperator>(Op0);
4741 BinaryOperator *BO1 = dyn_cast<BinaryOperator>(Op1);
4742 if (!BO0 && !BO1)
4743 return nullptr;
4744
4745 if (Instruction *NewICmp = foldICmpXNegX(I, Builder))
4746 return NewICmp;
4747
4748 const CmpInst::Predicate Pred = I.getPredicate();
4749 Value *X;
4750
4751 // Convert add-with-unsigned-overflow comparisons into a 'not' with compare.
4752 // (Op1 + X) u</u>= Op1 --> ~Op1 u</u>= X
4753 if (match(Op0, m_OneUse(m_c_Add(m_Specific(Op1), m_Value(X)))) &&
4754 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
4755 return new ICmpInst(Pred, Builder.CreateNot(Op1), X);
4756 // Op0 u>/u<= (Op0 + X) --> X u>/u<= ~Op0
4757 if (match(Op1, m_OneUse(m_c_Add(m_Specific(Op0), m_Value(X)))) &&
4758 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE))
4759 return new ICmpInst(Pred, X, Builder.CreateNot(Op0));
4760
4761 {
4762 // (Op1 + X) + C u</u>= Op1 --> ~C - X u</u>= Op1
4763 Constant *C;
4764 if (match(Op0, m_OneUse(m_Add(m_c_Add(m_Specific(Op1), m_Value(X)),
4765 m_ImmConstant(C)))) &&
4766 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) {
4768 return new ICmpInst(Pred, Builder.CreateSub(C2, X), Op1);
4769 }
4770 // Op0 u>/u<= (Op0 + X) + C --> Op0 u>/u<= ~C - X
4771 if (match(Op1, m_OneUse(m_Add(m_c_Add(m_Specific(Op0), m_Value(X)),
4772 m_ImmConstant(C)))) &&
4773 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) {
4775 return new ICmpInst(Pred, Op0, Builder.CreateSub(C2, X));
4776 }
4777 }
4778
4779 {
4780 // Similar to above: an unsigned overflow comparison may use offset + mask:
4781 // ((Op1 + C) & C) u< Op1 --> Op1 != 0
4782 // ((Op1 + C) & C) u>= Op1 --> Op1 == 0
4783 // Op0 u> ((Op0 + C) & C) --> Op0 != 0
4784 // Op0 u<= ((Op0 + C) & C) --> Op0 == 0
4785 BinaryOperator *BO;
4786 const APInt *C;
4787 if ((Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) &&
4788 match(Op0, m_And(m_BinOp(BO), m_LowBitMask(C))) &&
4790 CmpInst::Predicate NewPred =
4792 Constant *Zero = ConstantInt::getNullValue(Op1->getType());
4793 return new ICmpInst(NewPred, Op1, Zero);
4794 }
4795
4796 if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) &&
4797 match(Op1, m_And(m_BinOp(BO), m_LowBitMask(C))) &&
4799 CmpInst::Predicate NewPred =
4801 Constant *Zero = ConstantInt::getNullValue(Op1->getType());
4802 return new ICmpInst(NewPred, Op0, Zero);
4803 }
4804 }
4805
4806 bool NoOp0WrapProblem = false, NoOp1WrapProblem = false;
4807 bool Op0HasNUW = false, Op1HasNUW = false;
4808 bool Op0HasNSW = false, Op1HasNSW = false;
4809 // Analyze the case when either Op0 or Op1 is an add instruction.
4810 // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null).
4811 auto hasNoWrapProblem = [](const BinaryOperator &BO, CmpInst::Predicate Pred,
4812 bool &HasNSW, bool &HasNUW) -> bool {
4813 if (isa<OverflowingBinaryOperator>(BO)) {
4814 HasNUW = BO.hasNoUnsignedWrap();
4815 HasNSW = BO.hasNoSignedWrap();
4816 return ICmpInst::isEquality(Pred) ||
4817 (CmpInst::isUnsigned(Pred) && HasNUW) ||
4818 (CmpInst::isSigned(Pred) && HasNSW);
4819 } else if (BO.getOpcode() == Instruction::Or) {
4820 HasNUW = true;
4821 HasNSW = true;
4822 return true;
4823 } else {
4824 return false;
4825 }
4826 };
4827 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
4828
4829 if (BO0) {
4830 match(BO0, m_AddLike(m_Value(A), m_Value(B)));
4831 NoOp0WrapProblem = hasNoWrapProblem(*BO0, Pred, Op0HasNSW, Op0HasNUW);
4832 }
4833 if (BO1) {
4834 match(BO1, m_AddLike(m_Value(C), m_Value(D)));
4835 NoOp1WrapProblem = hasNoWrapProblem(*BO1, Pred, Op1HasNSW, Op1HasNUW);
4836 }
4837
4838 // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow.
4839 // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow.
4840 if ((A == Op1 || B == Op1) && NoOp0WrapProblem)
4841 return new ICmpInst(Pred, A == Op1 ? B : A,
4842 Constant::getNullValue(Op1->getType()));
4843
4844 // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow.
4845 // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow.
4846 if ((C == Op0 || D == Op0) && NoOp1WrapProblem)
4847 return new ICmpInst(Pred, Constant::getNullValue(Op0->getType()),
4848 C == Op0 ? D : C);
4849
4850 // icmp (A+B), (A+D) -> icmp B, D for equalities or if there is no overflow.
4851 if (A && C && (A == C || A == D || B == C || B == D) && NoOp0WrapProblem &&
4852 NoOp1WrapProblem) {
4853 // Determine Y and Z in the form icmp (X+Y), (X+Z).
4854 Value *Y, *Z;
4855 if (A == C) {
4856 // C + B == C + D -> B == D
4857 Y = B;
4858 Z = D;
4859 } else if (A == D) {
4860 // D + B == C + D -> B == C
4861 Y = B;
4862 Z = C;
4863 } else if (B == C) {
4864 // A + C == C + D -> A == D
4865 Y = A;
4866 Z = D;
4867 } else {
4868 assert(B == D);
4869 // A + D == C + D -> A == C
4870 Y = A;
4871 Z = C;
4872 }
4873 return new ICmpInst(Pred, Y, Z);
4874 }
4875
4876 // icmp slt (A + -1), Op1 -> icmp sle A, Op1
4877 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT &&
4878 match(B, m_AllOnes()))
4879 return new ICmpInst(CmpInst::ICMP_SLE, A, Op1);
4880
4881 // icmp sge (A + -1), Op1 -> icmp sgt A, Op1
4882 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE &&
4883 match(B, m_AllOnes()))
4884 return new ICmpInst(CmpInst::ICMP_SGT, A, Op1);
4885
4886 // icmp sle (A + 1), Op1 -> icmp slt A, Op1
4887 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(B, m_One()))
4888 return new ICmpInst(CmpInst::ICMP_SLT, A, Op1);
4889
4890 // icmp sgt (A + 1), Op1 -> icmp sge A, Op1
4891 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(B, m_One()))
4892 return new ICmpInst(CmpInst::ICMP_SGE, A, Op1);
4893
4894 // icmp sgt Op0, (C + -1) -> icmp sge Op0, C
4895 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT &&
4896 match(D, m_AllOnes()))
4897 return new ICmpInst(CmpInst::ICMP_SGE, Op0, C);
4898
4899 // icmp sle Op0, (C + -1) -> icmp slt Op0, C
4900 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE &&
4901 match(D, m_AllOnes()))
4902 return new ICmpInst(CmpInst::ICMP_SLT, Op0, C);
4903
4904 // icmp sge Op0, (C + 1) -> icmp sgt Op0, C
4905 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(D, m_One()))
4906 return new ICmpInst(CmpInst::ICMP_SGT, Op0, C);
4907
4908 // icmp slt Op0, (C + 1) -> icmp sle Op0, C
4909 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(D, m_One()))
4910 return new ICmpInst(CmpInst::ICMP_SLE, Op0, C);
4911
4912 // TODO: The subtraction-related identities shown below also hold, but
4913 // canonicalization from (X -nuw 1) to (X + -1) means that the combinations
4914 // wouldn't happen even if they were implemented.
4915 //
4916 // icmp ult (A - 1), Op1 -> icmp ule A, Op1
4917 // icmp uge (A - 1), Op1 -> icmp ugt A, Op1
4918 // icmp ugt Op0, (C - 1) -> icmp uge Op0, C
4919 // icmp ule Op0, (C - 1) -> icmp ult Op0, C
4920
4921 // icmp ule (A + 1), Op0 -> icmp ult A, Op1
4922 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(B, m_One()))
4923 return new ICmpInst(CmpInst::ICMP_ULT, A, Op1);
4924
4925 // icmp ugt (A + 1), Op0 -> icmp uge A, Op1
4926 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(B, m_One()))
4927 return new ICmpInst(CmpInst::ICMP_UGE, A, Op1);
4928
4929 // icmp uge Op0, (C + 1) -> icmp ugt Op0, C
4930 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(D, m_One()))
4931 return new ICmpInst(CmpInst::ICMP_UGT, Op0, C);
4932
4933 // icmp ult Op0, (C + 1) -> icmp ule Op0, C
4934 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(D, m_One()))
4935 return new ICmpInst(CmpInst::ICMP_ULE, Op0, C);
4936
4937 // if C1 has greater magnitude than C2:
4938 // icmp (A + C1), (C + C2) -> icmp (A + C3), C
4939 // s.t. C3 = C1 - C2
4940 //
4941 // if C2 has greater magnitude than C1:
4942 // icmp (A + C1), (C + C2) -> icmp A, (C + C3)
4943 // s.t. C3 = C2 - C1
4944 if (A && C && NoOp0WrapProblem && NoOp1WrapProblem &&
4945 (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) {
4946 const APInt *AP1, *AP2;
4947 // TODO: Support non-uniform vectors.
4948 // TODO: Allow undef passthrough if B AND D's element is undef.
4949 if (match(B, m_APIntAllowUndef(AP1)) && match(D, m_APIntAllowUndef(AP2)) &&
4950 AP1->isNegative() == AP2->isNegative()) {
4951 APInt AP1Abs = AP1->abs();
4952 APInt AP2Abs = AP2->abs();
4953 if (AP1Abs.uge(AP2Abs)) {
4954 APInt Diff = *AP1 - *AP2;
4955 Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff);
4956 Value *NewAdd = Builder.CreateAdd(
4957 A, C3, "", Op0HasNUW && Diff.ule(*AP1), Op0HasNSW);
4958 return new ICmpInst(Pred, NewAdd, C);
4959 } else {
4960 APInt Diff = *AP2 - *AP1;
4961 Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff);
4962 Value *NewAdd = Builder.CreateAdd(
4963 C, C3, "", Op1HasNUW && Diff.ule(*AP2), Op1HasNSW);
4964 return new ICmpInst(Pred, A, NewAdd);
4965 }
4966 }
4967 Constant *Cst1, *Cst2;
4968 if (match(B, m_ImmConstant(Cst1)) && match(D, m_ImmConstant(Cst2)) &&
4969 ICmpInst::isEquality(Pred)) {
4970 Constant *Diff = ConstantExpr::getSub(Cst2, Cst1);
4971 Value *NewAdd = Builder.CreateAdd(C, Diff);
4972 return new ICmpInst(Pred, A, NewAdd);
4973 }
4974 }
4975
4976 // Analyze the case when either Op0 or Op1 is a sub instruction.
4977 // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null).
4978 A = nullptr;
4979 B = nullptr;
4980 C = nullptr;
4981 D = nullptr;
4982 if (BO0 && BO0->getOpcode() == Instruction::Sub) {
4983 A = BO0->getOperand(0);
4984 B = BO0->getOperand(1);
4985 }
4986 if (BO1 && BO1->getOpcode() == Instruction::Sub) {
4987 C = BO1->getOperand(0);
4988 D = BO1->getOperand(1);
4989 }
4990
4991 // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow.
4992 if (A == Op1 && NoOp0WrapProblem)
4993 return new ICmpInst(Pred, Constant::getNullValue(Op1->getType()), B);
4994 // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow.
4995 if (C == Op0 && NoOp1WrapProblem)
4996 return new ICmpInst(Pred, D, Constant::getNullValue(Op0->getType()));
4997
4998 // Convert sub-with-unsigned-overflow comparisons into a comparison of args.
4999 // (A - B) u>/u<= A --> B u>/u<= A
5000 if (A == Op1 && (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE))
5001 return new ICmpInst(Pred, B, A);
5002 // C u</u>= (C - D) --> C u</u>= D
5003 if (C == Op0 && (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
5004 return new ICmpInst(Pred, C, D);
5005 // (A - B) u>=/u< A --> B u>/u<= A iff B != 0
5006 if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
5007 isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
5009 // C u<=/u> (C - D) --> C u</u>= D iff B != 0
5010 if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
5011 isKnownNonZero(D, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
5013
5014 // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow.
5015 if (B && D && B == D && NoOp0WrapProblem && NoOp1WrapProblem)
5016 return new ICmpInst(Pred, A, C);
5017
5018 // icmp (A-B), (A-D) -> icmp D, B for equalities or if there is no overflow.
5019 if (A && C && A == C && NoOp0WrapProblem && NoOp1WrapProblem)
5020 return new ICmpInst(Pred, D, B);
5021
5022 // icmp (0-X) < cst --> x > -cst
5023 if (NoOp0WrapProblem && ICmpInst::isSigned(Pred)) {
5024 Value *X;
5025 if (match(BO0, m_Neg(m_Value(X))))
5026 if (Constant *RHSC = dyn_cast<Constant>(Op1))
5027 if (RHSC->isNotMinSignedValue())
5028 return new ICmpInst(I.getSwappedPredicate(), X,
5029 ConstantExpr::getNeg(RHSC));
5030 }
5031
5032 if (Instruction * R = foldICmpXorXX(I, Q, *this))
5033 return R;
5034 if (Instruction *R = foldICmpOrXX(I, Q, *this))
5035 return R;
5036
5037 {
5038 // Try to remove shared multiplier from comparison:
5039 // X * Z u{lt/le/gt/ge}/eq/ne Y * Z
5040 Value *X, *Y, *Z;
5041 if (Pred == ICmpInst::getUnsignedPredicate(Pred) &&
5042 ((match(Op0, m_Mul(m_Value(X), m_Value(Z))) &&
5043 match(Op1, m_c_Mul(m_Specific(Z), m_Value(Y)))) ||
5044 (match(Op0, m_Mul(m_Value(Z), m_Value(X))) &&
5045 match(Op1, m_c_Mul(m_Specific(Z), m_Value(Y)))))) {
5046 bool NonZero;
5047 if (ICmpInst::isEquality(Pred)) {
5048 KnownBits ZKnown = computeKnownBits(Z, 0, &I);
5049 // if Z % 2 != 0
5050 // X * Z eq/ne Y * Z -> X eq/ne Y
5051 if (ZKnown.countMaxTrailingZeros() == 0)
5052 return new ICmpInst(Pred, X, Y);
5053 NonZero = !ZKnown.One.isZero() ||
5054 isKnownNonZero(Z, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT);
5055 // if Z != 0 and nsw(X * Z) and nsw(Y * Z)
5056 // X * Z eq/ne Y * Z -> X eq/ne Y
5057 if (NonZero && BO0 && BO1 && Op0HasNSW && Op1HasNSW)
5058 return new ICmpInst(Pred, X, Y);
5059 } else
5060 NonZero = isKnownNonZero(Z, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT);
5061
5062 // If Z != 0 and nuw(X * Z) and nuw(Y * Z)
5063 // X * Z u{lt/le/gt/ge}/eq/ne Y * Z -> X u{lt/le/gt/ge}/eq/ne Y
5064 if (NonZero && BO0 && BO1 && Op0HasNUW && Op1HasNUW)
5065 return new ICmpInst(Pred, X, Y);
5066 }
5067 }
5068
5069 BinaryOperator *SRem = nullptr;
5070 // icmp (srem X, Y), Y
5071 if (BO0 && BO0->getOpcode() == Instruction::SRem && Op1 == BO0->getOperand(1))
5072 SRem = BO0;
5073 // icmp Y, (srem X, Y)
5074 else if (BO1 && BO1->getOpcode() == Instruction::SRem &&
5075 Op0 == BO1->getOperand(1))
5076 SRem = BO1;
5077 if (SRem) {
5078 // We don't check hasOneUse to avoid increasing register pressure because
5079 // the value we use is the same value this instruction was already using.
5080 switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(Pred) : Pred) {
5081 default:
5082 break;
5083 case ICmpInst::ICMP_EQ:
5084 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5085 case ICmpInst::ICMP_NE:
5086 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5087 case ICmpInst::ICMP_SGT:
5088 case ICmpInst::ICMP_SGE:
5089 return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(1),
5091 case ICmpInst::ICMP_SLT:
5092 case ICmpInst::ICMP_SLE:
5093 return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(1),
5095 }
5096 }
5097
5098 if (BO0 && BO1 && BO0->getOpcode() == BO1->getOpcode() &&
5099 (BO0->hasOneUse() || BO1->hasOneUse()) &&
5100 BO0->getOperand(1) == BO1->getOperand(1)) {
5101 switch (BO0->getOpcode()) {
5102 default:
5103 break;
5104 case Instruction::Add:
5105 case Instruction::Sub:
5106 case Instruction::Xor: {
5107 if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b
5108 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5109
5110 const APInt *C;
5111 if (match(BO0->getOperand(1), m_APInt(C))) {
5112 // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b
5113 if (C->isSignMask()) {
5114 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate();
5115 return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0));
5116 }
5117
5118 // icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b
5119 if (BO0->getOpcode() == Instruction::Xor && C->isMaxSignedValue()) {
5120 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate();
5121 NewPred = I.getSwappedPredicate(NewPred);
5122 return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0));
5123 }
5124 }
5125 break;
5126 }
5127 case Instruction::Mul: {
5128 if (!I.isEquality())
5129 break;
5130
5131 const APInt *C;
5132 if (match(BO0->getOperand(1), m_APInt(C)) && !C->isZero() &&
5133 !C->isOne()) {
5134 // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask)
5135 // Mask = -1 >> count-trailing-zeros(C).
5136 if (unsigned TZs = C->countr_zero()) {
5137 Constant *Mask = ConstantInt::get(
5138 BO0->getType(),
5139 APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs));
5140 Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask);
5141 Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask);
5142 return new ICmpInst(Pred, And1, And2);
5143 }
5144 }
5145 break;
5146 }
5147 case Instruction::UDiv:
5148 case Instruction::LShr:
5149 if (I.isSigned() || !BO0->isExact() || !BO1->isExact())
5150 break;
5151 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5152
5153 case Instruction::SDiv:
5154 if (!(I.isEquality() || match(BO0->getOperand(1), m_NonNegative())) ||
5155 !BO0->isExact() || !BO1->isExact())
5156 break;
5157 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5158
5159 case Instruction::AShr:
5160 if (!BO0->isExact() || !BO1->isExact())
5161 break;
5162 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5163
5164 case Instruction::Shl: {
5165 bool NUW = Op0HasNUW && Op1HasNUW;
5166 bool NSW = Op0HasNSW && Op1HasNSW;
5167 if (!NUW && !NSW)
5168 break;
5169 if (!NSW && I.isSigned())
5170 break;
5171 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5172 }
5173 }
5174 }
5175
5176 if (BO0) {
5177 // Transform A & (L - 1) `ult` L --> L != 0
5178 auto LSubOne = m_Add(m_Specific(Op1), m_AllOnes());
5179 auto BitwiseAnd = m_c_And(m_Value(), LSubOne);
5180
5181 if (match(BO0, BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) {
5182 auto *Zero = Constant::getNullValue(BO0->getType());
5183 return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero);
5184 }
5185 }
5186
5187 // For unsigned predicates / eq / ne:
5188 // icmp pred (x << 1), x --> icmp getSignedPredicate(pred) x, 0
5189 // icmp pred x, (x << 1) --> icmp getSignedPredicate(pred) 0, x
5190 if (!ICmpInst::isSigned(Pred)) {
5191 if (match(Op0, m_Shl(m_Specific(Op1), m_One())))
5192 return new ICmpInst(ICmpInst::getSignedPredicate(Pred), Op1,
5193 Constant::getNullValue(Op1->getType()));
5194 else if (match(Op1, m_Shl(m_Specific(Op0), m_One())))
5195 return new ICmpInst(ICmpInst::getSignedPredicate(Pred),
5196 Constant::getNullValue(Op0->getType()), Op0);
5197 }
5198
5200 return replaceInstUsesWith(I, V);
5201
5202 if (Instruction *R = foldICmpAndXX(I, Q, *this))
5203 return R;
5204
5206 return replaceInstUsesWith(I, V);
5207
5209 return replaceInstUsesWith(I, V);
5210
5211 return nullptr;
5212}
5213
5214/// Fold icmp Pred min|max(X, Y), Z.
5217 Value *Z,
5218 ICmpInst::Predicate Pred) {
5219 Value *X = MinMax->getLHS();
5220 Value *Y = MinMax->getRHS();
5221 if (ICmpInst::isSigned(Pred) && !MinMax->isSigned())
5222 return nullptr;
5223 if (ICmpInst::isUnsigned(Pred) && MinMax->isSigned()) {
5224 // Revert the transform signed pred -> unsigned pred
5225 // TODO: We can flip the signedness of predicate if both operands of icmp
5226 // are negative.
5230 } else
5231 return nullptr;
5232 }
5234 auto IsCondKnownTrue = [](Value *Val) -> std::optional<bool> {
5235 if (!Val)
5236 return std::nullopt;
5237 if (match(Val, m_One()))
5238 return true;
5239 if (match(Val, m_Zero()))
5240 return false;
5241 return std::nullopt;
5242 };
5243 auto CmpXZ = IsCondKnownTrue(simplifyICmpInst(Pred, X, Z, Q));
5244 auto CmpYZ = IsCondKnownTrue(simplifyICmpInst(Pred, Y, Z, Q));
5245 if (!CmpXZ.has_value() && !CmpYZ.has_value())
5246 return nullptr;
5247 if (!CmpXZ.has_value()) {
5248 std::swap(X, Y);
5249 std::swap(CmpXZ, CmpYZ);
5250 }
5251
5252 auto FoldIntoCmpYZ = [&]() -> Instruction * {
5253 if (CmpYZ.has_value())
5254 return replaceInstUsesWith(I, ConstantInt::getBool(I.getType(), *CmpYZ));
5255 return ICmpInst::Create(Instruction::ICmp, Pred, Y, Z);
5256 };
5257
5258 switch (Pred) {
5259 case ICmpInst::ICMP_EQ:
5260 case ICmpInst::ICMP_NE: {
5261 // If X == Z:
5262 // Expr Result
5263 // min(X, Y) == Z X <= Y
5264 // max(X, Y) == Z X >= Y
5265 // min(X, Y) != Z X > Y
5266 // max(X, Y) != Z X < Y
5267 if ((Pred == ICmpInst::ICMP_EQ) == *CmpXZ) {
5268 ICmpInst::Predicate NewPred =
5269 ICmpInst::getNonStrictPredicate(MinMax->getPredicate());
5270 if (Pred == ICmpInst::ICMP_NE)
5271 NewPred = ICmpInst::getInversePredicate(NewPred);
5272 return ICmpInst::Create(Instruction::ICmp, NewPred, X, Y);
5273 }
5274 // Otherwise (X != Z):
5275 ICmpInst::Predicate NewPred = MinMax->getPredicate();
5276 auto MinMaxCmpXZ = IsCondKnownTrue(simplifyICmpInst(NewPred, X, Z, Q));
5277 if (!MinMaxCmpXZ.has_value()) {
5278 std::swap(X, Y);
5279 std::swap(CmpXZ, CmpYZ);
5280 // Re-check pre-condition X != Z
5281 if (!CmpXZ.has_value() || (Pred == ICmpInst::ICMP_EQ) == *CmpXZ)
5282 break;
5283 MinMaxCmpXZ = IsCondKnownTrue(simplifyICmpInst(NewPred, X, Z, Q));
5284 }
5285 if (!MinMaxCmpXZ.has_value())
5286 break;
5287 if (*MinMaxCmpXZ) {
5288 // Expr Fact Result
5289 // min(X, Y) == Z X < Z false
5290 // max(X, Y) == Z X > Z false
5291 // min(X, Y) != Z X < Z true
5292 // max(X, Y) != Z X > Z true
5293 return replaceInstUsesWith(
5294 I, ConstantInt::getBool(I.getType(), Pred == ICmpInst::ICMP_NE));
5295 } else {
5296 // Expr Fact Result
5297 // min(X, Y) == Z X > Z Y == Z
5298 // max(X, Y) == Z X < Z Y == Z
5299 // min(X, Y) != Z X > Z Y != Z
5300 // max(X, Y) != Z X < Z Y != Z
5301 return FoldIntoCmpYZ();
5302 }
5303 break;
5304 }
5305 case ICmpInst::ICMP_SLT:
5306 case ICmpInst::ICMP_ULT:
5307 case ICmpInst::ICMP_SLE:
5308 case ICmpInst::ICMP_ULE:
5309 case ICmpInst::ICMP_SGT:
5310 case ICmpInst::ICMP_UGT:
5311 case ICmpInst::ICMP_SGE:
5312 case ICmpInst::ICMP_UGE: {
5313 bool IsSame = MinMax->getPredicate() == ICmpInst::getStrictPredicate(Pred);
5314 if (*CmpXZ) {
5315 if (IsSame) {
5316 // Expr Fact Result
5317 // min(X, Y) < Z X < Z true
5318 // min(X, Y) <= Z X <= Z true
5319 // max(X, Y) > Z X > Z true
5320 // max(X, Y) >= Z X >= Z true
5321 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5322 } else {
5323 // Expr Fact Result
5324 // max(X, Y) < Z X < Z Y < Z
5325 // max(X, Y) <= Z X <= Z Y <= Z
5326 // min(X, Y) > Z X > Z Y > Z
5327 // min(X, Y) >= Z X >= Z Y >= Z
5328 return FoldIntoCmpYZ();
5329 }
5330 } else {
5331 if (IsSame) {
5332 // Expr Fact Result
5333 // min(X, Y) < Z X >= Z Y < Z
5334 // min(X, Y) <= Z X > Z Y <= Z
5335 // max(X, Y) > Z X <= Z Y > Z
5336 // max(X, Y) >= Z X < Z Y >= Z
5337 return FoldIntoCmpYZ();
5338 } else {
5339 // Expr Fact Result
5340 // max(X, Y) < Z X >= Z false
5341 // max(X, Y) <= Z X > Z false
5342 // min(X, Y) > Z X <= Z false
5343 // min(X, Y) >= Z X < Z false
5344 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5345 }
5346 }
5347 break;
5348 }
5349 default:
5350 break;
5351 }
5352
5353 return nullptr;
5354}
5355
5356// Canonicalize checking for a power-of-2-or-zero value:
5358 InstCombiner::BuilderTy &Builder) {
5359 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
5360 const CmpInst::Predicate Pred = I.getPredicate();
5361 Value *A = nullptr;
5362 bool CheckIs;
5363 if (I.isEquality()) {
5364 // (A & (A-1)) == 0 --> ctpop(A) < 2 (two commuted variants)
5365 // ((A-1) & A) != 0 --> ctpop(A) > 1 (two commuted variants)
5366 if (!match(Op0, m_OneUse(m_c_And(m_Add(m_Value(A), m_AllOnes()),
5367 m_Deferred(A)))) ||
5368 !match(Op1, m_ZeroInt()))
5369 A = nullptr;
5370
5371 // (A & -A) == A --> ctpop(A) < 2 (four commuted variants)
5372 // (-A & A) != A --> ctpop(A) > 1 (four commuted variants)
5373 if (match(Op0, m_OneUse(m_c_And(m_Neg(m_Specific(Op1)), m_Specific(Op1)))))
5374 A = Op1;
5375 else if (match(Op1,
5377 A = Op0;
5378
5379 CheckIs = Pred == ICmpInst::ICMP_EQ;
5380 } else if (ICmpInst::isUnsigned(Pred)) {
5381 // (A ^ (A-1)) u>= A --> ctpop(A) < 2 (two commuted variants)
5382 // ((A-1) ^ A) u< A --> ctpop(A) > 1 (two commuted variants)
5383
5384 if ((Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
5386 m_Specific(Op1))))) {
5387 A = Op1;
5388 CheckIs = Pred == ICmpInst::ICMP_UGE;
5389 } else if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) &&
5391 m_Specific(Op0))))) {
5392 A = Op0;
5393 CheckIs = Pred == ICmpInst::ICMP_ULE;
5394 }
5395 }
5396
5397 if (A) {
5398 Type *Ty = A->getType();
5399 CallInst *CtPop = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, A);
5400 return CheckIs ? new ICmpInst(ICmpInst::ICMP_ULT, CtPop,
5401 ConstantInt::get(Ty, 2))
5402 : new ICmpInst(ICmpInst::ICMP_UGT, CtPop,
5403 ConstantInt::get(Ty, 1));
5404 }
5405
5406 return nullptr;
5407}
5408
5410 if (!I.isEquality())
5411 return nullptr;
5412
5413 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
5414 const CmpInst::Predicate Pred = I.getPredicate();
5415 Value *A, *B, *C, *D;
5416 if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) {
5417 if (A == Op1 || B == Op1) { // (A^B) == A -> B == 0
5418 Value *OtherVal = A == Op1 ? B : A;
5419 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType()));
5420 }
5421
5422 if (match(Op1, m_Xor(m_Value(C), m_Value(D)))) {
5423 // A^c1 == C^c2 --> A == C^(c1^c2)
5424 ConstantInt *C1, *C2;
5425 if (match(B, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2)) &&
5426 Op1->hasOneUse()) {
5427 Constant *NC = Builder.getInt(C1->getValue() ^ C2->getValue());
5429 return new ICmpInst(Pred, A, Xor);
5430 }
5431
5432 // A^B == A^D -> B == D
5433 if (A == C)
5434 return new ICmpInst(Pred, B, D);
5435 if (A == D)
5436 return new ICmpInst(Pred, B, C);
5437 if (B == C)
5438 return new ICmpInst(Pred, A, D);
5439 if (B == D)
5440 return new ICmpInst(Pred, A, C);
5441 }
5442 }
5443
5444 // canoncalize:
5445 // (icmp eq/ne (and X, C), X)
5446 // -> (icmp eq/ne (and X, ~C), 0)
5447 {
5448 Constant *CMask;
5449 A = nullptr;
5450 if (match(Op0, m_OneUse(m_And(m_Specific(Op1), m_ImmConstant(CMask)))))
5451 A = Op1;
5452 else if (match(Op1, m_OneUse(m_And(m_Specific(Op0), m_ImmConstant(CMask)))))
5453 A = Op0;
5454 if (A)
5455 return new ICmpInst(Pred, Builder.CreateAnd(A, Builder.CreateNot(CMask)),
5456 Constant::getNullValue(A->getType()));
5457 }
5458
5459 if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && (A == Op0 || B == Op0)) {
5460 // A == (A^B) -> B == 0
5461 Value *OtherVal = A == Op0 ? B : A;
5462 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType()));
5463 }
5464
5465 // (X&Z) == (Y&Z) -> (X^Y) & Z == 0
5466 if (match(Op0, m_OneUse(m_And(m_Value(A), m_Value(B)))) &&
5467 match(Op1, m_OneUse(m_And(m_Value(C), m_Value(D))))) {
5468 Value *X = nullptr, *Y = nullptr, *Z = nullptr;
5469
5470 if (A == C) {
5471 X = B;
5472 Y = D;
5473 Z = A;
5474 } else if (A == D) {
5475 X = B;
5476 Y = C;
5477 Z = A;
5478 } else if (B == C) {
5479 X = A;
5480 Y = D;
5481 Z = B;
5482 } else if (B == D) {
5483 X = A;
5484 Y = C;
5485 Z = B;
5486 }
5487
5488 if (X) { // Build (X^Y) & Z
5489 Op1 = Builder.CreateXor(X, Y);
5490 Op1 = Builder.CreateAnd(Op1, Z);
5491 return new ICmpInst(Pred, Op1, Constant::getNullValue(Op1->getType()));
5492 }
5493 }
5494
5495 {
5496 // Similar to above, but specialized for constant because invert is needed:
5497 // (X | C) == (Y | C) --> (X ^ Y) & ~C == 0
5498 Value *X, *Y;
5499 Constant *C;
5500 if (match(Op0, m_OneUse(m_Or(m_Value(X), m_Constant(C)))) &&
5501 match(Op1, m_OneUse(m_Or(m_Value(Y), m_Specific(C))))) {
5504 return new ICmpInst(Pred, And, Constant::getNullValue(And->getType()));
5505 }
5506 }
5507
5508 if (match(Op1, m_ZExt(m_Value(A))) &&
5509 (Op0->hasOneUse() || Op1->hasOneUse())) {
5510 // (B & (Pow2C-1)) == zext A --> A == trunc B
5511 // (B & (Pow2C-1)) != zext A --> A != trunc B
5512 const APInt *MaskC;
5513 if (match(Op0, m_And(m_Value(B), m_LowBitMask(MaskC))) &&
5514 MaskC->countr_one() == A->getType()->getScalarSizeInBits())
5515 return new ICmpInst(Pred, A, Builder.CreateTrunc(B, A->getType()));
5516 }
5517
5518 // (A >> C) == (B >> C) --> (A^B) u< (1 << C)
5519 // For lshr and ashr pairs.
5520 const APInt *AP1, *AP2;
5521 if ((match(Op0, m_OneUse(m_LShr(m_Value(A), m_APIntAllowUndef(AP1)))) &&
5522 match(Op1, m_OneUse(m_LShr(m_Value(B), m_APIntAllowUndef(AP2))))) ||
5523 (match(Op0, m_OneUse(m_AShr(m_Value(A), m_APIntAllowUndef(AP1)))) &&
5524 match(Op1, m_OneUse(m_AShr(m_Value(B), m_APIntAllowUndef(AP2)))))) {
5525 if (AP1 != AP2)
5526 return nullptr;
5527 unsigned TypeBits = AP1->getBitWidth();
5528 unsigned ShAmt = AP1->getLimitedValue(TypeBits);
5529 if (ShAmt < TypeBits && ShAmt != 0) {
5530 ICmpInst::Predicate NewPred =
5532 Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted");
5533 APInt CmpVal = APInt::getOneBitSet(TypeBits, ShAmt);
5534 return new ICmpInst(NewPred, Xor, ConstantInt::get(A->getType(), CmpVal));
5535 }
5536 }
5537
5538 // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0
5539 ConstantInt *Cst1;
5540 if (match(Op0, m_OneUse(m_Shl(m_Value(A), m_ConstantInt(Cst1)))) &&
5541 match(Op1, m_OneUse(m_Shl(m_Value(B), m_Specific(Cst1))))) {
5542 unsigned TypeBits = Cst1->getBitWidth();
5543 unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits);
5544 if (ShAmt < TypeBits && ShAmt != 0) {
5545 Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted");
5546 APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt);
5548 I.getName() + ".mask");
5549 return new ICmpInst(Pred, And, Constant::getNullValue(Cst1->getType()));
5550 }
5551 }
5552
5553 // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to
5554 // "icmp (and X, mask), cst"
5555 uint64_t ShAmt = 0;
5556 if (Op0->hasOneUse() &&
5557 match(Op0, m_Trunc(m_OneUse(m_LShr(m_Value(A), m_ConstantInt(ShAmt))))) &&
5558 match(Op1, m_ConstantInt(Cst1)) &&
5559 // Only do this when A has multiple uses. This is most important to do
5560 // when it exposes other optimizations.
5561 !A->hasOneUse()) {
5562 unsigned ASize = cast<IntegerType>(A->getType())->getPrimitiveSizeInBits();
5563
5564 if (ShAmt < ASize) {
5565 APInt MaskV =
5567 MaskV <<= ShAmt;
5568
5569 APInt CmpV = Cst1->getValue().zext(ASize);
5570 CmpV <<= ShAmt;
5571
5572 Value *Mask = Builder.CreateAnd(A, Builder.getInt(MaskV));
5573 return new ICmpInst(Pred, Mask, Builder.getInt(CmpV));
5574 }
5575 }
5576
5578 return ICmp;
5579
5580 // Match icmp eq (trunc (lshr A, BW), (ashr (trunc A), BW-1)), which checks the
5581 // top BW/2 + 1 bits are all the same. Create "A >=s INT_MIN && A <=s INT_MAX",
5582 // which we generate as "icmp ult (add A, 2^(BW-1)), 2^BW" to skip a few steps
5583 // of instcombine.
5584 unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
5585 if (match(Op0, m_AShr(m_Trunc(m_Value(A)), m_SpecificInt(BitWidth - 1))) &&
5587 A->getType()->getScalarSizeInBits() == BitWidth * 2 &&
5588 (I.getOperand(0)->hasOneUse() || I.getOperand(1)->hasOneUse())) {
5590 Value *Add = Builder.CreateAdd(A, ConstantInt::get(A->getType(), C));
5591 return new ICmpInst(Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT
5593 Add, ConstantInt::get(A->getType(), C.shl(1)));
5594 }
5595
5596 // Canonicalize:
5597 // Assume B_Pow2 != 0
5598 // 1. A & B_Pow2 != B_Pow2 -> A & B_Pow2 == 0
5599 // 2. A & B_Pow2 == B_Pow2 -> A & B_Pow2 != 0
5600 if (match(Op0, m_c_And(m_Specific(Op1), m_Value())) &&
5601 isKnownToBeAPowerOfTwo(Op1, /* OrZero */ false, 0, &I))
5602 return new ICmpInst(CmpInst::getInversePredicate(Pred), Op0,
5604
5605 if (match(Op1, m_c_And(m_Specific(Op0), m_Value())) &&
5606 isKnownToBeAPowerOfTwo(Op0, /* OrZero */ false, 0, &I))
5607 return new ICmpInst(CmpInst::getInversePredicate(Pred), Op1,
5608 ConstantInt::getNullValue(Op1->getType()));
5609
5610 // Canonicalize:
5611 // icmp eq/ne X, OneUse(rotate-right(X))
5612 // -> icmp eq/ne X, rotate-left(X)
5613 // We generally try to convert rotate-right -> rotate-left, this just
5614 // canonicalizes another case.
5615 CmpInst::Predicate PredUnused = Pred;
5616 if (match(&I, m_c_ICmp(PredUnused, m_Value(A),
5617 m_OneUse(m_Intrinsic<Intrinsic::fshr>(
5618 m_Deferred(A), m_Deferred(A), m_Value(B))))))
5619 return new ICmpInst(
5620 Pred, A,
5621 Builder.CreateIntrinsic(Op0->getType(), Intrinsic::fshl, {A, A, B}));
5622
5623 // Canonicalize:
5624 // icmp eq/ne OneUse(A ^ Cst), B --> icmp eq/ne (A ^ B), Cst
5625 Constant *Cst;
5626 if (match(&I, m_c_ICmp(PredUnused,
5629 return new ICmpInst(Pred, Builder.CreateXor(A, B), Cst);
5630
5631 {
5632 // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2)
5633 auto m_Matcher =
5636 m_Sub(m_Value(B), m_Deferred(A)));
5637 std::optional<bool> IsZero = std::nullopt;
5638 if (match(&I, m_c_ICmp(PredUnused, m_OneUse(m_c_And(m_Value(A), m_Matcher)),
5639 m_Deferred(A))))
5640 IsZero = false;
5641 // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0)
5642 else if (match(&I,
5643 m_ICmp(PredUnused, m_OneUse(m_c_And(m_Value(A), m_Matcher)),
5644 m_Zero())))
5645 IsZero = true;
5646
5647 if (IsZero && isKnownToBeAPowerOfTwo(A, /* OrZero */ true, /*Depth*/ 0, &I))
5648 // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2)
5649 // -> (icmp eq/ne (and X, P2), 0)
5650 // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0)
5651 // -> (icmp eq/ne (and X, P2), P2)
5652 return new ICmpInst(Pred, Builder.CreateAnd(B, A),
5653 *IsZero ? A
5654 : ConstantInt::getNullValue(A->getType()));
5655 }
5656
5657 return nullptr;
5658}
5659
5661 ICmpInst::Predicate Pred = ICmp.getPredicate();
5662 Value *Op0 = ICmp.getOperand(0), *Op1 = ICmp.getOperand(1);
5663
5664 // Try to canonicalize trunc + compare-to-constant into a mask + cmp.
5665 // The trunc masks high bits while the compare may effectively mask low bits.
5666 Value *X;
5667 const APInt *C;
5668 if (!match(Op0, m_OneUse(m_Trunc(m_Value(X)))) || !match(Op1, m_APInt(C)))
5669 return nullptr;
5670
5671 // This matches patterns corresponding to tests of the signbit as well as:
5672 // (trunc X) u< C --> (X & -C) == 0 (are all masked-high-bits clear?)
5673 // (trunc X) u> C --> (X & ~C) != 0 (are any masked-high-bits set?)
5674 APInt Mask;
5675 if (decomposeBitTestICmp(Op0, Op1, Pred, X, Mask, true /* WithTrunc */)) {
5676 Value *And = Builder.CreateAnd(X, Mask);
5677 Constant *Zero = ConstantInt::getNullValue(X->getType());
5678 return new ICmpInst(Pred, And, Zero);
5679 }
5680
5681 unsigned SrcBits = X->getType()->getScalarSizeInBits();
5682 if (Pred == ICmpInst::ICMP_ULT && C->isNegatedPowerOf2()) {
5683 // If C is a negative power-of-2 (high-bit mask):
5684 // (trunc X) u< C --> (X & C) != C (are any masked-high-bits clear?)
5685 Constant *MaskC = ConstantInt::get(X->getType(), C->zext(SrcBits));
5686 Value *And = Builder.CreateAnd(X, MaskC);
5687 return new ICmpInst(ICmpInst::ICMP_NE, And, MaskC);
5688 }
5689
5690 if (Pred == ICmpInst::ICMP_UGT && (~*C).isPowerOf2()) {
5691 // If C is not-of-power-of-2 (one clear bit):
5692 // (trunc X) u> C --> (X & (C+1)) == C+1 (are all masked-high-bits set?)
5693 Constant *MaskC = ConstantInt::get(X->getType(), (*C + 1).zext(SrcBits));
5694 Value *And = Builder.CreateAnd(X, MaskC);
5695 return new ICmpInst(ICmpInst::ICMP_EQ, And, MaskC);
5696 }
5697
5698 if (auto *II = dyn_cast<IntrinsicInst>(X)) {
5699 if (II->getIntrinsicID() == Intrinsic::cttz ||
5700 II->getIntrinsicID() == Intrinsic::ctlz) {
5701 unsigned MaxRet = SrcBits;
5702 // If the "is_zero_poison" argument is set, then we know at least
5703 // one bit is set in the input, so the result is always at least one
5704 // less than the full bitwidth of that input.
5705 if (match(II->getArgOperand(1), m_One()))
5706 MaxRet--;
5707
5708 // Make sure the destination is wide enough to hold the largest output of
5709 // the intrinsic.
5710 if (llvm::Log2_32(MaxRet) + 1 <= Op0->getType()->getScalarSizeInBits())
5711 if (Instruction *I =
5712 foldICmpIntrinsicWithConstant(ICmp, II, C->zext(SrcBits)))
5713 return I;
5714 }
5715 }
5716
5717 return nullptr;
5718}
5719
5721 assert(isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0");
5722 auto *CastOp0 = cast<CastInst>(ICmp.getOperand(0));
5723 Value *X;
5724 if (!match(CastOp0, m_ZExtOrSExt(m_Value(X))))
5725 return nullptr;
5726
5727 bool IsSignedExt = CastOp0->getOpcode() == Instruction::SExt;
5728 bool IsSignedCmp = ICmp.isSigned();
5729
5730 // icmp Pred (ext X), (ext Y)
5731 Value *Y;
5732 if (match(ICmp.getOperand(1), m_ZExtOrSExt(m_Value(Y)))) {
5733 bool IsZext0 = isa<ZExtInst>(ICmp.getOperand(0));
5734 bool IsZext1 = isa<ZExtInst>(ICmp.getOperand(1));
5735
5736 if (IsZext0 != IsZext1) {
5737 // If X and Y and both i1
5738 // (icmp eq/ne (zext X) (sext Y))
5739 // eq -> (icmp eq (or X, Y), 0)
5740 // ne -> (icmp ne (or X, Y), 0)
5741 if (ICmp.isEquality() && X->getType()->isIntOrIntVectorTy(1) &&
5742 Y->getType()->isIntOrIntVectorTy(1))
5743 return new ICmpInst(ICmp.getPredicate(), Builder.CreateOr(X, Y),
5744 Constant::getNullValue(X->getType()));
5745
5746 // If we have mismatched casts and zext has the nneg flag, we can
5747 // treat the "zext nneg" as "sext". Otherwise, we cannot fold and quit.
5748
5749 auto *NonNegInst0 = dyn_cast<PossiblyNonNegInst>(ICmp.getOperand(0));
5750 auto *NonNegInst1 = dyn_cast<PossiblyNonNegInst>(ICmp.getOperand(1));
5751
5752 bool IsNonNeg0 = NonNegInst0 && NonNegInst0->hasNonNeg();
5753 bool IsNonNeg1 = NonNegInst1 && NonNegInst1->hasNonNeg();
5754
5755 if ((IsZext0 && IsNonNeg0) || (IsZext1 && IsNonNeg1))
5756 IsSignedExt = true;
5757 else
5758 return nullptr;
5759 }
5760
5761 // Not an extension from the same type?
5762 Type *XTy = X->getType(), *YTy = Y->getType();
5763 if (XTy != YTy) {
5764 // One of the casts must have one use because we are creating a new cast.
5765 if (!ICmp.getOperand(0)->hasOneUse() && !ICmp.getOperand(1)->hasOneUse())
5766 return nullptr;
5767 // Extend the narrower operand to the type of the wider operand.
5768 CastInst::CastOps CastOpcode =
5769 IsSignedExt ? Instruction::SExt : Instruction::ZExt;
5770 if (XTy->getScalarSizeInBits() < YTy->getScalarSizeInBits())
5771 X = Builder.CreateCast(CastOpcode, X, YTy);
5772 else if (YTy->getScalarSizeInBits() < XTy->getScalarSizeInBits())
5773 Y = Builder.CreateCast(CastOpcode, Y, XTy);
5774 else
5775 return nullptr;
5776 }
5777
5778 // (zext X) == (zext Y) --> X == Y
5779 // (sext X) == (sext Y) --> X == Y
5780 if (ICmp.isEquality())
5781 return new ICmpInst(ICmp.getPredicate(), X, Y);
5782
5783 // A signed comparison of sign extended values simplifies into a
5784 // signed comparison.
5785 if (IsSignedCmp && IsSignedExt)
5786 return new ICmpInst(ICmp.getPredicate(), X, Y);
5787
5788 // The other three cases all fold into an unsigned comparison.
5789 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Y);
5790 }
5791
5792 // Below here, we are only folding a compare with constant.
5793 auto *C = dyn_cast<Constant>(ICmp.getOperand(1));
5794 if (!C)
5795 return nullptr;
5796
5797 // If a lossless truncate is possible...
5798 Type *SrcTy = CastOp0->getSrcTy();
5799 Constant *Res = getLosslessTrunc(C, SrcTy, CastOp0->getOpcode());
5800 if (Res) {
5801 if (ICmp.isEquality())
5802 return new ICmpInst(ICmp.getPredicate(), X, Res);
5803
5804 // A signed comparison of sign extended values simplifies into a
5805 // signed comparison.
5806 if (IsSignedExt && IsSignedCmp)
5807 return new ICmpInst(ICmp.getPredicate(), X, Res);
5808
5809 // The other three cases all fold into an unsigned comparison.
5810 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Res);
5811 }
5812
5813 // The re-extended constant changed, partly changed (in the case of a vector),
5814 // or could not be determined to be equal (in the case of a constant
5815 // expression), so the constant cannot be represented in the shorter type.
5816 // All the cases that fold to true or false will have already been handled
5817 // by simplifyICmpInst, so only deal with the tricky case.
5818 if (IsSignedCmp || !IsSignedExt || !isa<ConstantInt>(C))
5819 return nullptr;
5820
5821 // Is source op positive?
5822 // icmp ult (sext X), C --> icmp sgt X, -1
5823 if (ICmp.getPredicate() == ICmpInst::ICMP_ULT)
5825
5826 // Is source op negative?
5827 // icmp ugt (sext X), C --> icmp slt X, 0
5828 assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!");
5830}
5831
5832/// Handle icmp (cast x), (cast or constant).
5834 // If any operand of ICmp is a inttoptr roundtrip cast then remove it as
5835 // icmp compares only pointer's value.
5836 // icmp (inttoptr (ptrtoint p1)), p2 --> icmp p1, p2.
5837 Value *SimplifiedOp0 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(0));
5838 Value *SimplifiedOp1 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(1));
5839 if (SimplifiedOp0 || SimplifiedOp1)
5840 return new ICmpInst(ICmp.getPredicate(),
5841 SimplifiedOp0 ? SimplifiedOp0 : ICmp.getOperand(0),
5842 SimplifiedOp1 ? SimplifiedOp1 : ICmp.getOperand(1));
5843
5844 auto *CastOp0 = dyn_cast<CastInst>(ICmp.getOperand(0));
5845 if (!CastOp0)
5846 return nullptr;
5847 if (!isa<Constant>(ICmp.getOperand(1)) && !isa<CastInst>(ICmp.getOperand(1)))
5848 return nullptr;
5849
5850 Value *Op0Src = CastOp0->getOperand(0);
5851 Type *SrcTy = CastOp0->getSrcTy();
5852 Type *DestTy = CastOp0->getDestTy();
5853
5854 // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the
5855 // integer type is the same size as the pointer type.
5856 auto CompatibleSizes = [&](Type *SrcTy, Type *DestTy) {
5857 if (isa<VectorType>(SrcTy)) {
5858 SrcTy = cast<VectorType>(SrcTy)->getElementType();
5859 DestTy = cast<VectorType>(DestTy)->getElementType();
5860 }
5861 return DL.getPointerTypeSizeInBits(SrcTy) == DestTy->getIntegerBitWidth();
5862 };
5863 if (CastOp0->getOpcode() == Instruction::PtrToInt &&
5864 CompatibleSizes(SrcTy, DestTy)) {
5865 Value *NewOp1 = nullptr;
5866 if (auto *PtrToIntOp1 = dyn_cast<PtrToIntOperator>(ICmp.getOperand(1))) {
5867 Value *PtrSrc = PtrToIntOp1->getOperand(0);
5868 if (PtrSrc->getType() == Op0Src->getType())
5869 NewOp1 = PtrToIntOp1->getOperand(0);
5870 } else if (auto *RHSC = dyn_cast<Constant>(ICmp.getOperand(1))) {
5871 NewOp1 = ConstantExpr::getIntToPtr(RHSC, SrcTy);
5872 }
5873
5874 if (NewOp1)
5875 return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1);
5876 }
5877
5878 if (Instruction *R = foldICmpWithTrunc(ICmp))
5879 return R;
5880
5881 return foldICmpWithZextOrSext(ICmp);
5882}
5883
5884static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS, bool IsSigned) {
5885 switch (BinaryOp) {
5886 default:
5887 llvm_unreachable("Unsupported binary op");
5888 case Instruction::Add:
5889 case Instruction::Sub:
5890 return match(RHS, m_Zero());
5891 case Instruction::Mul:
5892 return !(RHS->getType()->isIntOrIntVectorTy(1) && IsSigned) &&
5893 match(RHS, m_One());
5894 }
5895}
5896
5899 bool IsSigned, Value *LHS, Value *RHS,
5900 Instruction *CxtI) const {
5901 switch (BinaryOp) {
5902 default:
5903 llvm_unreachable("Unsupported binary op");
5904 case Instruction::Add:
5905 if (IsSigned)
5906 return computeOverflowForSignedAdd(LHS, RHS, CxtI);
5907 else
5908 return computeOverflowForUnsignedAdd(LHS, RHS, CxtI);
5909 case Instruction::Sub:
5910 if (IsSigned)
5911 return computeOverflowForSignedSub(LHS, RHS, CxtI);
5912 else
5913 return computeOverflowForUnsignedSub(LHS, RHS, CxtI);
5914 case Instruction::Mul:
5915 if (IsSigned)
5916 return computeOverflowForSignedMul(LHS, RHS, CxtI);
5917 else
5918 return computeOverflowForUnsignedMul(LHS, RHS, CxtI);
5919 }
5920}
5921
5922bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp,
5923 bool IsSigned, Value *LHS,
5924 Value *RHS, Instruction &OrigI,
5925 Value *&Result,
5926 Constant *&Overflow) {
5927 if (OrigI.isCommutative() && isa<Constant>(LHS) && !isa<Constant>(RHS))
5928 std::swap(LHS, RHS);
5929
5930 // If the overflow check was an add followed by a compare, the insertion point
5931 // may be pointing to the compare. We want to insert the new instructions
5932 // before the add in case there are uses of the add between the add and the
5933 // compare.
5934 Builder.SetInsertPoint(&OrigI);
5935
5936 Type *OverflowTy = Type::getInt1Ty(LHS->getContext());
5937 if (auto *LHSTy = dyn_cast<VectorType>(LHS->getType()))
5938 OverflowTy = VectorType::get(OverflowTy, LHSTy->getElementCount());
5939
5940 if (isNeutralValue(BinaryOp, RHS, IsSigned)) {
5941 Result = LHS;
5942 Overflow = ConstantInt::getFalse(OverflowTy);
5943 return true;
5944 }
5945
5946 switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, &OrigI)) {
5948 return false;
5951 Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
5952 Result->takeName(&OrigI);
5953 Overflow = ConstantInt::getTrue(OverflowTy);
5954 return true;
5956 Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
5957 Result->takeName(&OrigI);
5958 Overflow = ConstantInt::getFalse(OverflowTy);
5959 if (auto *Inst = dyn_cast<Instruction>(Result)) {
5960 if (IsSigned)
5961 Inst->setHasNoSignedWrap();
5962 else
5963 Inst->setHasNoUnsignedWrap();
5964 }
5965 return true;
5966 }
5967
5968 llvm_unreachable("Unexpected overflow result");
5969}
5970
5971/// Recognize and process idiom involving test for multiplication
5972/// overflow.
5973///
5974/// The caller has matched a pattern of the form:
5975/// I = cmp u (mul(zext A, zext B), V
5976/// The function checks if this is a test for overflow and if so replaces
5977/// multiplication with call to 'mul.with.overflow' intrinsic.
5978///
5979/// \param I Compare instruction.
5980/// \param MulVal Result of 'mult' instruction. It is one of the arguments of
5981/// the compare instruction. Must be of integer type.
5982/// \param OtherVal The other argument of compare instruction.
5983/// \returns Instruction which must replace the compare instruction, NULL if no
5984/// replacement required.
5986 const APInt *OtherVal,
5987 InstCombinerImpl &IC) {
5988 // Don't bother doing this transformation for pointers, don't do it for
5989 // vectors.
5990 if (!isa<IntegerType>(MulVal->getType()))
5991 return nullptr;
5992
5993 auto *MulInstr = dyn_cast<Instruction>(MulVal);
5994 if (!MulInstr)
5995 return nullptr;
5996 assert(MulInstr->getOpcode() == Instruction::Mul);
5997
5998 auto *LHS = cast<ZExtInst>(MulInstr->getOperand(0)),
5999 *RHS = cast<ZExtInst>(MulInstr->getOperand(1));
6000 assert(LHS->getOpcode() == Instruction::ZExt);
6001 assert(RHS->getOpcode() == Instruction::ZExt);
6002 Value *A = LHS->getOperand(0), *B = RHS->getOperand(0);
6003
6004 // Calculate type and width of the result produced by mul.with.overflow.
6005 Type *TyA = A->getType(), *TyB = B->getType();
6006 unsigned WidthA = TyA->getPrimitiveSizeInBits(),
6007 WidthB = TyB->getPrimitiveSizeInBits();
6008 unsigned MulWidth;
6009 Type *MulType;
6010 if (WidthB > WidthA) {
6011 MulWidth = WidthB;
6012 MulType = TyB;
6013 } else {
6014 MulWidth = WidthA;
6015 MulType = TyA;
6016 }
6017
6018 // In order to replace the original mul with a narrower mul.with.overflow,
6019 // all uses must ignore upper bits of the product. The number of used low
6020 // bits must be not greater than the width of mul.with.overflow.
6021 if (MulVal->hasNUsesOrMore(2))
6022 for (User *U : MulVal->users()) {
6023 if (U == &I)
6024 continue;
6025 if (TruncInst *TI = dyn_cast<TruncInst>(U)) {
6026 // Check if truncation ignores bits above MulWidth.
6027 unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits();
6028 if (TruncWidth > MulWidth)
6029 return nullptr;
6030 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) {
6031 // Check if AND ignores bits above MulWidth.
6032 if (BO->getOpcode() != Instruction::And)
6033 return nullptr;
6034 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->getOperand(1))) {
6035 const APInt &CVal = CI->getValue();
6036 if (CVal.getBitWidth() - CVal.countl_zero() > MulWidth)
6037 return nullptr;
6038 } else {
6039 // In this case we could have the operand of the binary operation
6040 // being defined in another block, and performing the replacement
6041 // could break the dominance relation.
6042 return nullptr;
6043 }
6044 } else {
6045 // Other uses prohibit this transformation.
6046 return nullptr;
6047 }
6048 }
6049
6050 // Recognize patterns
6051 switch (I.getPredicate()) {
6052 case ICmpInst::ICMP_UGT: {
6053 // Recognize pattern:
6054 // mulval = mul(zext A, zext B)
6055 // cmp ugt mulval, max
6056 APInt MaxVal = APInt::getMaxValue(MulWidth);
6057 MaxVal = MaxVal.zext(OtherVal->getBitWidth());
6058 if (MaxVal.eq(*OtherVal))
6059 break; // Recognized
6060 return nullptr;
6061 }
6062
6063 case ICmpInst::ICMP_ULT: {
6064 // Recognize pattern:
6065 // mulval = mul(zext A, zext B)
6066 // cmp ule mulval, max + 1
6067 APInt MaxVal = APInt::getOneBitSet(OtherVal->getBitWidth(), MulWidth);
6068 if (MaxVal.eq(*OtherVal))
6069 break; // Recognized
6070 return nullptr;
6071 }
6072
6073 default:
6074 return nullptr;
6075 }
6076
6077 InstCombiner::BuilderTy &Builder = IC.Builder;
6078 Builder.SetInsertPoint(MulInstr);
6079
6080 // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B)
6081 Value *MulA = A, *MulB = B;
6082 if (WidthA < MulWidth)
6083 MulA = Builder.CreateZExt(A, MulType);
6084 if (WidthB < MulWidth)
6085 MulB = Builder.CreateZExt(B, MulType);
6087 I.getModule(), Intrinsic::umul_with_overflow, MulType);
6088 CallInst *Call = Builder.CreateCall(F, {MulA, MulB}, "umul");
6089 IC.addToWorklist(MulInstr);
6090
6091 // If there are uses of mul result other than the comparison, we know that
6092 // they are truncation or binary AND. Change them to use result of
6093 // mul.with.overflow and adjust properly mask/size.
6094 if (MulVal->hasNUsesOrMore(2)) {
6095 Value *Mul = Builder.CreateExtractValue(Call, 0, "umul.value");
6096 for (User *U : make_early_inc_range(MulVal->users())) {
6097 if (U == &I)
6098 continue;
6099 if (TruncInst *TI = dyn_cast<TruncInst>(U)) {
6100 if (TI->getType()->getPrimitiveSizeInBits() == MulWidth)
6101 IC.replaceInstUsesWith(*TI, Mul);
6102 else
6103 TI->setOperand(0, Mul);
6104 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) {
6105 assert(BO->getOpcode() == Instruction::And);
6106 // Replace (mul & mask) --> zext (mul.with.overflow & short_mask)
6107 ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1));
6108 APInt ShortMask = CI->getValue().trunc(MulWidth);
6109 Value *ShortAnd = Builder.CreateAnd(Mul, ShortMask);
6110 Value *Zext = Builder.CreateZExt(ShortAnd, BO->getType());
6111 IC.replaceInstUsesWith(*BO, Zext);
6112 } else {
6113 llvm_unreachable("Unexpected Binary operation");
6114 }
6115 IC.addToWorklist(cast<Instruction>(U));
6116 }
6117 }
6118
6119 // The original icmp gets replaced with the overflow value, maybe inverted
6120 // depending on predicate.
6121 if (I.getPredicate() == ICmpInst::ICMP_ULT) {
6122 Value *Res = Builder.CreateExtractValue(Call, 1);
6123 return BinaryOperator::CreateNot(Res);
6124 }
6125
6126 return ExtractValueInst::Create(Call, 1);
6127}
6128
6129/// When performing a comparison against a constant, it is possible that not all
6130/// the bits in the LHS are demanded. This helper method computes the mask that
6131/// IS demanded.
6133 const APInt *RHS;
6134 if (!match(I.getOperand(1), m_APInt(RHS)))
6136
6137 // If this is a normal comparison, it demands all bits. If it is a sign bit
6138 // comparison, it only demands the sign bit.
6139 bool UnusedBit;
6140 if (isSignBitCheck(I.getPredicate(), *RHS, UnusedBit))
6142
6143 switch (I.getPredicate()) {
6144 // For a UGT comparison, we don't care about any bits that
6145 // correspond to the trailing ones of the comparand. The value of these
6146 // bits doesn't impact the outcome of the comparison, because any value
6147 // greater than the RHS must differ in a bit higher than these due to carry.
6148 case ICmpInst::ICMP_UGT:
6149 return APInt::getBitsSetFrom(BitWidth, RHS->countr_one());
6150
6151 // Similarly, for a ULT comparison, we don't care about the trailing zeros.
6152 // Any value less than the RHS must differ in a higher bit because of carries.
6153 case ICmpInst::ICMP_ULT:
6154 return APInt::getBitsSetFrom(BitWidth, RHS->countr_zero());
6155
6156 default:
6158 }
6159}
6160
6161/// Check that one use is in the same block as the definition and all
6162/// other uses are in blocks dominated by a given block.
6163///
6164/// \param DI Definition
6165/// \param UI Use
6166/// \param DB Block that must dominate all uses of \p DI outside
6167/// the parent block
6168/// \return true when \p UI is the only use of \p DI in the parent block
6169/// and all other uses of \p DI are in blocks dominated by \p DB.
6170///
6172 const Instruction *UI,
6173 const BasicBlock *DB) const {
6174 assert(DI && UI && "Instruction not defined\n");
6175 // Ignore incomplete definitions.
6176 if (!DI->getParent())
6177 return false;
6178 // DI and UI must be in the same block.
6179 if (DI->getParent() != UI->getParent())
6180 return false;
6181 // Protect from self-referencing blocks.
6182 if (DI->getParent() == DB)
6183 return false;
6184 for (const User *U : DI->users()) {
6185 auto *Usr = cast<Instruction>(U);
6186 if (Usr != UI && !DT.dominates(DB, Usr->getParent()))
6187 return false;
6188 }
6189 return true;
6190}
6191
6192/// Return true when the instruction sequence within a block is select-cmp-br.
6193static bool isChainSelectCmpBranch(const SelectInst *SI) {
6194 const BasicBlock *BB = SI->getParent();
6195 if (!BB)
6196 return false;
6197 auto *BI = dyn_cast_or_null<BranchInst>(BB->getTerminator());
6198 if (!BI || BI->getNumSuccessors() != 2)
6199 return false;
6200 auto *IC = dyn_cast<ICmpInst>(BI->getCondition());
6201 if (!IC || (IC->getOperand(0) != SI && IC->getOperand(1) != SI))
6202 return false;
6203 return true;
6204}
6205
6206/// True when a select result is replaced by one of its operands
6207/// in select-icmp sequence. This will eventually result in the elimination
6208/// of the select.
6209///
6210/// \param SI Select instruction
6211/// \param Icmp Compare instruction
6212/// \param SIOpd Operand that replaces the select
6213///
6214/// Notes:
6215/// - The replacement is global and requires dominator information
6216/// - The caller is responsible for the actual replacement
6217///
6218/// Example:
6219///
6220/// entry:
6221/// %4 = select i1 %3, %C* %0, %C* null
6222/// %5 = icmp eq %C* %4, null
6223/// br i1 %5, label %9, label %7
6224/// ...
6225/// ; <label>:7 ; preds = %entry
6226/// %8 = getelementptr inbounds %C* %4, i64 0, i32 0
6227/// ...
6228///
6229/// can be transformed to
6230///
6231/// %5 = icmp eq %C* %0, null
6232/// %6 = select i1 %3, i1 %5, i1 true
6233/// br i1 %6, label %9, label %7
6234/// ...
6235/// ; <label>:7 ; preds = %entry
6236/// %8 = getelementptr inbounds %C* %0, i64 0, i32 0 // replace by %0!
6237///
6238/// Similar when the first operand of the select is a constant or/and
6239/// the compare is for not equal rather than equal.
6240///
6241/// NOTE: The function is only called when the select and compare constants
6242/// are equal, the optimization can work only for EQ predicates. This is not a
6243/// major restriction since a NE compare should be 'normalized' to an equal
6244/// compare, which usually happens in the combiner and test case
6245/// select-cmp-br.ll checks for it.
6247 const ICmpInst *Icmp,
6248 const unsigned SIOpd) {
6249 assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!");
6251 BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(1);
6252 // The check for the single predecessor is not the best that can be
6253 // done. But it protects efficiently against cases like when SI's
6254 // home block has two successors, Succ and Succ1, and Succ1 predecessor
6255 // of Succ. Then SI can't be replaced by SIOpd because the use that gets
6256 // replaced can be reached on either path. So the uniqueness check
6257 // guarantees that the path all uses of SI (outside SI's parent) are on
6258 // is disjoint from all other paths out of SI. But that information
6259 // is more expensive to compute, and the trade-off here is in favor
6260 // of compile-time. It should also be noticed that we check for a single
6261 // predecessor and not only uniqueness. This to handle the situation when
6262 // Succ and Succ1 points to the same basic block.
6263 if (Succ->getSinglePredecessor() && dominatesAllUses(SI, Icmp, Succ)) {
6264 NumSel++;
6265 SI->replaceUsesOutsideBlock(SI->getOperand(SIOpd), SI->getParent());
6266 return true;
6267 }
6268 }
6269 return false;
6270}
6271
6272/// Try to fold the comparison based on range information we can get by checking
6273/// whether bits are known to be zero or one in the inputs.
6275 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
6276 Type *Ty = Op0->getType();
6277 ICmpInst::Predicate Pred = I.getPredicate();
6278
6279 // Get scalar or pointer size.
6280 unsigned BitWidth = Ty->isIntOrIntVectorTy()
6281 ? Ty->getScalarSizeInBits()
6283
6284 if (!BitWidth)
6285 return nullptr;
6286
6287 KnownBits Op0Known(BitWidth);
6288 KnownBits Op1Known(BitWidth);
6289
6290 {
6291 // Don't use dominating conditions when folding icmp using known bits. This
6292 // may convert signed into unsigned predicates in ways that other passes
6293 // (especially IndVarSimplify) may not be able to reliably undo.
6294 SQ.DC = nullptr;
6295 auto _ = make_scope_exit([&]() { SQ.DC = &DC; });
6297 Op0Known, 0))
6298 return &I;
6299
6300 if (SimplifyDemandedBits(&I, 1, APInt::getAllOnes(BitWidth), Op1Known, 0))
6301 return &I;
6302 }
6303
6304 // Given the known and unknown bits, compute a range that the LHS could be
6305 // in. Compute the Min, Max and RHS values based on the known bits. For the
6306 // EQ and NE we use unsigned values.
6307 APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0);
6308 APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0);
6309 if (I.isSigned()) {
6310 Op0Min = Op0Known.getSignedMinValue();
6311 Op0Max = Op0Known.getSignedMaxValue();
6312 Op1Min = Op1Known.getSignedMinValue();
6313 Op1Max = Op1Known.getSignedMaxValue();
6314 } else {
6315 Op0Min = Op0Known.getMinValue();
6316 Op0Max = Op0Known.getMaxValue();
6317 Op1Min = Op1Known.getMinValue();
6318 Op1Max = Op1Known.getMaxValue();
6319 }
6320
6321 // If Min and Max are known to be the same, then SimplifyDemandedBits figured
6322 // out that the LHS or RHS is a constant. Constant fold this now, so that
6323 // code below can assume that Min != Max.
6324 if (!isa<Constant>(Op0) && Op0Min == Op0Max)
6325 return new ICmpInst(Pred, ConstantExpr::getIntegerValue(Ty, Op0Min), Op1);
6326 if (!isa<Constant>(Op1) && Op1Min == Op1Max)
6327 return new ICmpInst(Pred, Op0, ConstantExpr::getIntegerValue(Ty, Op1Min));
6328
6329 // Don't break up a clamp pattern -- (min(max X, Y), Z) -- by replacing a
6330 // min/max canonical compare with some other compare. That could lead to
6331 // conflict with select canonicalization and infinite looping.
6332 // FIXME: This constraint may go away if min/max intrinsics are canonical.
6333 auto isMinMaxCmp = [&](Instruction &Cmp) {
6334 if (!Cmp.hasOneUse())
6335 return false;
6336 Value *A, *B;
6337 SelectPatternFlavor SPF = matchSelectPattern(Cmp.user_back(), A, B).Flavor;
6339 return false;
6340 return match(Op0, m_MaxOrMin(m_Value(), m_Value())) ||
6341 match(Op1, m_MaxOrMin(m_Value(), m_Value()));
6342 };
6343 if (!isMinMaxCmp(I)) {
6344 switch (Pred) {
6345 default:
6346 break;
6347 case ICmpInst::ICMP_ULT: {
6348 if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B)
6349 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6350 const APInt *CmpC;
6351 if (match(Op1, m_APInt(CmpC))) {
6352 // A <u C -> A == C-1 if min(A)+1 == C
6353 if (*CmpC == Op0Min + 1)
6354 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6355 ConstantInt::get(Op1->getType(), *CmpC - 1));
6356 // X <u C --> X == 0, if the number of zero bits in the bottom of X
6357 // exceeds the log2 of C.
6358 if (Op0Known.countMinTrailingZeros() >= CmpC->ceilLogBase2())
6359 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6360 Constant::getNullValue(Op1->getType()));
6361 }
6362 break;
6363 }
6364 case ICmpInst::ICMP_UGT: {
6365 if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B)
6366 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6367 const APInt *CmpC;
6368 if (match(Op1, m_APInt(CmpC))) {
6369 // A >u C -> A == C+1 if max(a)-1 == C
6370 if (*CmpC == Op0Max - 1)
6371 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6372 ConstantInt::get(Op1->getType(), *CmpC + 1));
6373 // X >u C --> X != 0, if the number of zero bits in the bottom of X
6374 // exceeds the log2 of C.
6375 if (Op0Known.countMinTrailingZeros() >= CmpC->getActiveBits())
6376 return new ICmpInst(ICmpInst::ICMP_NE, Op0,
6377 Constant::getNullValue(Op1->getType()));
6378 }
6379 break;
6380 }
6381 case ICmpInst::ICMP_SLT: {
6382 if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B)
6383 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6384 const APInt *CmpC;
6385 if (match(Op1, m_APInt(CmpC))) {
6386 if (*CmpC == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C
6387 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6388 ConstantInt::get(Op1->getType(), *CmpC - 1));
6389 }
6390 break;
6391 }
6392 case ICmpInst::ICMP_SGT: {
6393 if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B)
6394 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6395 const APInt *CmpC;
6396 if (match(Op1, m_APInt(CmpC))) {
6397 if (*CmpC == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C
6398 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6399 ConstantInt::get(Op1->getType(), *CmpC + 1));
6400 }
6401 break;
6402 }
6403 }
6404 }
6405
6406 // Based on the range information we know about the LHS, see if we can
6407 // simplify this comparison. For example, (x&4) < 8 is always true.
6408 switch (Pred) {
6409 default:
6410 llvm_unreachable("Unknown icmp opcode!");
6411 case ICmpInst::ICMP_EQ:
6412 case ICmpInst::ICMP_NE: {
6413 if (Op0Max.ult(Op1Min) || Op0Min.ugt(Op1Max))
6414 return replaceInstUsesWith(
6415 I, ConstantInt::getBool(I.getType(), Pred == CmpInst::ICMP_NE));
6416
6417 // If all bits are known zero except for one, then we know at most one bit
6418 // is set. If the comparison is against zero, then this is a check to see if
6419 // *that* bit is set.
6420 APInt Op0KnownZeroInverted = ~Op0Known.Zero;
6421 if (Op1Known.isZero()) {
6422 // If the LHS is an AND with the same constant, look through it.
6423 Value *LHS = nullptr;
6424 const APInt *LHSC;
6425 if (!match(Op0, m_And(m_Value(LHS), m_APInt(LHSC))) ||
6426 *LHSC != Op0KnownZeroInverted)
6427 LHS = Op0;
6428
6429 Value *X;
6430 const APInt *C1;
6431 if (match(LHS, m_Shl(m_Power2(C1), m_Value(X)))) {
6432 Type *XTy = X->getType();
6433 unsigned Log2C1 = C1->countr_zero();
6434 APInt C2 = Op0KnownZeroInverted;
6435 APInt C2Pow2 = (C2 & ~(*C1 - 1)) + *C1;
6436 if (C2Pow2.isPowerOf2()) {
6437 // iff (C1 is pow2) & ((C2 & ~(C1-1)) + C1) is pow2):
6438 // ((C1 << X) & C2) == 0 -> X >= (Log2(C2+C1) - Log2(C1))
6439 // ((C1 << X) & C2) != 0 -> X < (Log2(C2+C1) - Log2(C1))
6440 unsigned Log2C2 = C2Pow2.countr_zero();
6441 auto *CmpC = ConstantInt::get(XTy, Log2C2 - Log2C1);
6442 auto NewPred =
6444 return new ICmpInst(NewPred, X, CmpC);
6445 }
6446 }
6447 }
6448
6449 // Op0 eq C_Pow2 -> Op0 ne 0 if Op0 is known to be C_Pow2 or zero.
6450 if (Op1Known.isConstant() && Op1Known.getConstant().isPowerOf2() &&
6451 (Op0Known & Op1Known) == Op0Known)
6452 return new ICmpInst(CmpInst::getInversePredicate(Pred), Op0,
6453 ConstantInt::getNullValue(Op1->getType()));
6454 break;
6455 }
6456 case ICmpInst::ICMP_ULT: {
6457 if (Op0Max.ult(Op1Min)) // A <u B -> true if max(A) < min(B)
6458 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
6459 if (Op0Min.uge(Op1Max)) // A <u B -> false if min(A) >= max(B)
6460 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
6461 break;
6462 }
6463 case ICmpInst::ICMP_UGT: {
6464 if (Op0Min.ugt(Op1Max)) // A >u B -> true if min(A) > max(B)
6465 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
6466 if (Op0Max.ule(Op1Min)) // A >u B -> false if max(A) <= max(B)
6467 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
6468 break;
6469 }
6470 case ICmpInst::ICMP_SLT: {
6471 if (Op0Max.slt(Op1Min)) // A <s B -> true if max(A) < min(C)
6472 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
6473 if (Op0Min.sge(Op1Max)) // A <s B -> false if min(A) >= max(C)
6474 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
6475 break;
6476 }
6477 case ICmpInst::ICMP_SGT: {
6478 if (Op0Min.sgt(Op1Max)) // A >s B -> true if min(A) > max(B)
6479 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
6480 if (Op0Max.sle(Op1Min)) // A >s B -> false if max(A) <= min(B)
6481 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
6482 break;
6483 }
6484 case ICmpInst::ICMP_SGE:
6485 assert(!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!");
6486 if (Op0Min.sge(Op1Max)) // A >=s B -> true if min(A) >= max(B)
6487 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
6488 if (Op0Max.slt(Op1Min)) // A >=s B -> false if max(A) < min(B)
6489 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
6490 if (Op1Min == Op0Max) // A >=s B -> A == B if max(A) == min(B)
6491 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6492 break;
6493 case ICmpInst::ICMP_SLE:
6494 assert(!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!");
6495 if (Op0Max.sle(Op1Min)) // A <=s B -> true if max(A) <= min(B)
6496 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
6497 if (Op0Min.sgt(Op1Max)) // A <=s B -> false if min(A) > max(B)
6498 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
6499 if (Op1Max == Op0Min) // A <=s B -> A == B if min(A) == max(B)
6500 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6501 break;
6502 case ICmpInst::ICMP_UGE:
6503 assert(!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!");
6504 if (Op0Min.uge(Op1Max)) // A >=u B -> true if min(A) >= max(B)
6505 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
6506 if (Op0Max.ult(Op1Min)) // A >=u B -> false if max(A) < min(B)
6507 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
6508 if (Op1Min == Op0Max) // A >=u B -> A == B if max(A) == min(B)
6509 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6510 break;
6511 case ICmpInst::ICMP_ULE:
6512 assert(!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!");
6513 if (Op0Max.ule(Op1Min)) // A <=u B -> true if max(A) <= min(B)
6514 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
6515 if (Op0Min.ugt(Op1Max)) // A <=u B -> false if min(A) > max(B)
6516 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
6517 if (Op1Max == Op0Min) // A <=u B -> A == B if min(A) == max(B)
6518 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6519 break;
6520 }
6521
6522 // Turn a signed comparison into an unsigned one if both operands are known to
6523 // have the same sign.
6524 if (I.isSigned() &&
6525 ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) ||
6526 (Op0Known.One.isNegative() && Op1Known.One.isNegative())))
6527 return new ICmpInst(I.getUnsignedPredicate(), Op0, Op1);
6528
6529 return nullptr;
6530}
6531
6532/// If one operand of an icmp is effectively a bool (value range of {0,1}),
6533/// then try to reduce patterns based on that limit.
6535 Value *X, *Y;
6537
6538 // X must be 0 and bool must be true for "ULT":
6539 // X <u (zext i1 Y) --> (X == 0) & Y
6540 if (match(&I, m_c_ICmp(Pred, m_Value(X), m_OneUse(m_ZExt(m_Value(Y))))) &&
6541 Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULT)
6542 return BinaryOperator::CreateAnd(Builder.CreateIsNull(X), Y);
6543
6544 // X must be 0 or bool must be true for "ULE":
6545 // X <=u (sext i1 Y) --> (X == 0) | Y
6546 if (match(&I, m_c_ICmp(Pred, m_Value(X), m_OneUse(m_SExt(m_Value(Y))))) &&
6547 Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULE)
6548 return BinaryOperator::CreateOr(Builder.CreateIsNull(X), Y);
6549
6550 // icmp eq/ne X, (zext/sext (icmp eq/ne X, C))
6551 ICmpInst::Predicate Pred1, Pred2;
6552 const APInt *C;
6553 Instruction *ExtI;
6554 if (match(&I, m_c_ICmp(Pred1, m_Value(X),
6557 m_APInt(C)))))) &&
6558 ICmpInst::isEquality(Pred1) && ICmpInst::isEquality(Pred2)) {
6559 bool IsSExt = ExtI->getOpcode() == Instruction::SExt;
6560 bool HasOneUse = ExtI->hasOneUse() && ExtI->getOperand(0)->hasOneUse();
6561 auto CreateRangeCheck = [&] {
6562 Value *CmpV1 =
6563 Builder.CreateICmp(Pred1, X, Constant::getNullValue(X->getType()));
6564 Value *CmpV2 = Builder.CreateICmp(
6565 Pred1, X, ConstantInt::getSigned(X->getType(), IsSExt ? -1 : 1));
6567 Pred1 == ICmpInst::ICMP_EQ ? Instruction::Or : Instruction::And,
6568 CmpV1, CmpV2);
6569 };
6570 if (C->isZero()) {
6571 if (Pred2 == ICmpInst::ICMP_EQ) {
6572 // icmp eq X, (zext/sext (icmp eq X, 0)) --> false
6573 // icmp ne X, (zext/sext (icmp eq X, 0)) --> true
6574 return replaceInstUsesWith(
6575 I, ConstantInt::getBool(I.getType(), Pred1 == ICmpInst::ICMP_NE));
6576 } else if (!IsSExt || HasOneUse) {
6577 // icmp eq X, (zext (icmp ne X, 0)) --> X == 0 || X == 1
6578 // icmp ne X, (zext (icmp ne X, 0)) --> X != 0 && X != 1
6579 // icmp eq X, (sext (icmp ne X, 0)) --> X == 0 || X == -1
6580 // icmp ne X, (sext (icmp ne X, 0)) --> X != 0 && X == -1
6581 return CreateRangeCheck();
6582 }
6583 } else if (IsSExt ? C->isAllOnes() : C->isOne()) {
6584 if (Pred2 == ICmpInst::ICMP_NE) {
6585 // icmp eq X, (zext (icmp ne X, 1)) --> false
6586 // icmp ne X, (zext (icmp ne X, 1)) --> true
6587 // icmp eq X, (sext (icmp ne X, -1)) --> false
6588 // icmp ne X, (sext (icmp ne X, -1)) --> true
6589 return replaceInstUsesWith(
6590 I, ConstantInt::getBool(I.getType(), Pred1 == ICmpInst::ICMP_NE));
6591 } else if (!IsSExt || HasOneUse) {
6592 // icmp eq X, (zext (icmp eq X, 1)) --> X == 0 || X == 1
6593 // icmp ne X, (zext (icmp eq X, 1)) --> X != 0 && X != 1
6594 // icmp eq X, (sext (icmp eq X, -1)) --> X == 0 || X == -1
6595 // icmp ne X, (sext (icmp eq X, -1)) --> X != 0 && X == -1
6596 return CreateRangeCheck();
6597 }
6598 } else {
6599 // when C != 0 && C != 1:
6600 // icmp eq X, (zext (icmp eq X, C)) --> icmp eq X, 0
6601 // icmp eq X, (zext (icmp ne X, C)) --> icmp eq X, 1
6602 // icmp ne X, (zext (icmp eq X, C)) --> icmp ne X, 0
6603 // icmp ne X, (zext (icmp ne X, C)) --> icmp ne X, 1
6604 // when C != 0 && C != -1:
6605 // icmp eq X, (sext (icmp eq X, C)) --> icmp eq X, 0
6606 // icmp eq X, (sext (icmp ne X, C)) --> icmp eq X, -1
6607 // icmp ne X, (sext (icmp eq X, C)) --> icmp ne X, 0
6608 // icmp ne X, (sext (icmp ne X, C)) --> icmp ne X, -1
6609 return ICmpInst::Create(
6610 Instruction::ICmp, Pred1, X,
6611 ConstantInt::getSigned(X->getType(), Pred2 == ICmpInst::ICMP_NE
6612 ? (IsSExt ? -1 : 1)
6613 : 0));
6614 }
6615 }
6616
6617 return nullptr;
6618}
6619
6620std::optional<std::pair<CmpInst::Predicate, Constant *>>
6622 Constant *C) {
6624 "Only for relational integer predicates.");
6625
6626 Type *Type = C->getType();
6627 bool IsSigned = ICmpInst::isSigned(Pred);
6628
6630 bool WillIncrement =
6631 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT;
6632
6633 // Check if the constant operand can be safely incremented/decremented
6634 // without overflowing/underflowing.
6635 auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) {
6636 return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned);
6637 };
6638
6639 Constant *SafeReplacementConstant = nullptr;
6640 if (auto *CI = dyn_cast<ConstantInt>(C)) {
6641 // Bail out if the constant can't be safely incremented/decremented.
6642 if (!ConstantIsOk(CI))
6643 return std::nullopt;
6644 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Type)) {
6645 unsigned NumElts = FVTy->getNumElements();
6646 for (unsigned i = 0; i != NumElts; ++i) {
6647 Constant *Elt = C->getAggregateElement(i);
6648 if (!Elt)
6649 return std::nullopt;
6650
6651 if (isa<UndefValue>(Elt))
6652 continue;
6653
6654 // Bail out if we can't determine if this constant is min/max or if we
6655 // know that this constant is min/max.
6656 auto *CI = dyn_cast<ConstantInt>(Elt);
6657 if (!CI || !ConstantIsOk(CI))
6658 return std::nullopt;
6659
6660 if (!SafeReplacementConstant)
6661 SafeReplacementConstant = CI;
6662 }
6663 } else if (isa<VectorType>(C->getType())) {
6664 // Handle scalable splat
6665 Value *SplatC = C->getSplatValue();
6666 auto *CI = dyn_cast_or_null<ConstantInt>(SplatC);
6667 // Bail out if the constant can't be safely incremented/decremented.
6668 if (!CI || !ConstantIsOk(CI))
6669 return std::nullopt;
6670 } else {
6671 // ConstantExpr?
6672 return std::nullopt;
6673 }
6674
6675 // It may not be safe to change a compare predicate in the presence of
6676 // undefined elements, so replace those elements with the first safe constant
6677 // that we found.
6678 // TODO: in case of poison, it is safe; let's replace undefs only.
6679 if (C->containsUndefOrPoisonElement()) {
6680 assert(SafeReplacementConstant && "Replacement constant not set");
6681 C = Constant::replaceUndefsWith(C, SafeReplacementConstant);
6682 }
6683
6685
6686 // Increment or decrement the constant.
6687 Constant *OneOrNegOne = ConstantInt::get(Type, WillIncrement ? 1 : -1, true);
6688 Constant *NewC = ConstantExpr::getAdd(C, OneOrNegOne);
6689
6690 return std::make_pair(NewPred, NewC);
6691}
6692
6693/// If we have an icmp le or icmp ge instruction with a constant operand, turn
6694/// it into the appropriate icmp lt or icmp gt instruction. This transform
6695/// allows them to be folded in visitICmpInst.
6697 ICmpInst::Predicate Pred = I.getPredicate();
6698 if (ICmpInst::isEquality(Pred) || !ICmpInst::isIntPredicate(Pred) ||
6700 return nullptr;
6701
6702 Value *Op0 = I.getOperand(0);
6703 Value *Op1 = I.getOperand(1);
6704 auto *Op1C = dyn_cast<Constant>(Op1);
6705 if (!Op1C)
6706 return nullptr;
6707
6708 auto FlippedStrictness =
6710 if (!FlippedStrictness)
6711 return nullptr;
6712
6713 return new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second);
6714}
6715
6716/// If we have a comparison with a non-canonical predicate, if we can update
6717/// all the users, invert the predicate and adjust all the users.
6719 // Is the predicate already canonical?
6720 CmpInst::Predicate Pred = I.getPredicate();
6722 return nullptr;
6723
6724 // Can all users be adjusted to predicate inversion?
6725 if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr))
6726 return nullptr;
6727
6728 // Ok, we can canonicalize comparison!
6729 // Let's first invert the comparison's predicate.
6730 I.setPredicate(CmpInst::getInversePredicate(Pred));
6731 I.setName(I.getName() + ".not");
6732
6733 // And, adapt users.
6735
6736 return &I;
6737}
6738
6739/// Integer compare with boolean values can always be turned into bitwise ops.
6741 InstCombiner::BuilderTy &Builder) {
6742 Value *A = I.getOperand(0), *B = I.getOperand(1);
6743 assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only");
6744
6745 // A boolean compared to true/false can be simplified to Op0/true/false in
6746 // 14 out of the 20 (10 predicates * 2 constants) possible combinations.
6747 // Cases not handled by InstSimplify are always 'not' of Op0.
6748 if (match(B, m_Zero())) {
6749 switch (I.getPredicate()) {
6750 case CmpInst::ICMP_EQ: // A == 0 -> !A
6751 case CmpInst::ICMP_ULE: // A <=u 0 -> !A
6752 case CmpInst::ICMP_SGE: // A >=s 0 -> !A
6754 default:
6755 llvm_unreachable("ICmp i1 X, C not simplified as expected.");
6756 }
6757 } else if (match(B, m_One())) {
6758 switch (I.getPredicate()) {
6759 case CmpInst::ICMP_NE: // A != 1 -> !A
6760 case CmpInst::ICMP_ULT: // A <u 1 -> !A
6761 case CmpInst::ICMP_SGT: // A >s -1 -> !A
6763 default:
6764 llvm_unreachable("ICmp i1 X, C not simplified as expected.");
6765 }
6766 }
6767
6768 switch (I.getPredicate()) {
6769 default:
6770 llvm_unreachable("Invalid icmp instruction!");
6771 case ICmpInst::ICMP_EQ:
6772 // icmp eq i1 A, B -> ~(A ^ B)
6773 return BinaryOperator::CreateNot(Builder.CreateXor(A, B));
6774
6775 case ICmpInst::ICMP_NE:
6776 // icmp ne i1 A, B -> A ^ B
6777 return BinaryOperator::CreateXor(A, B);
6778
6779 case ICmpInst::ICMP_UGT:
6780 // icmp ugt -> icmp ult
6781 std::swap(A, B);
6782 [[fallthrough]];
6783 case ICmpInst::ICMP_ULT:
6784 // icmp ult i1 A, B -> ~A & B
6785 return BinaryOperator::CreateAnd(Builder.CreateNot(A), B);
6786
6787 case ICmpInst::ICMP_SGT:
6788 // icmp sgt -> icmp slt
6789 std::swap(A, B);
6790 [[fallthrough]];
6791 case ICmpInst::ICMP_SLT:
6792 // icmp slt i1 A, B -> A & ~B
6793 return BinaryOperator::CreateAnd(Builder.CreateNot(B), A);
6794
6795 case ICmpInst::ICMP_UGE:
6796 // icmp uge -> icmp ule
6797 std::swap(A, B);
6798 [[fallthrough]];
6799 case ICmpInst::ICMP_ULE:
6800 // icmp ule i1 A, B -> ~A | B
6801 return BinaryOperator::CreateOr(Builder.CreateNot(A), B);
6802
6803 case ICmpInst::ICMP_SGE:
6804 // icmp sge -> icmp sle
6805 std::swap(A, B);
6806 [[fallthrough]];
6807 case ICmpInst::ICMP_SLE:
6808 // icmp sle i1 A, B -> A | ~B
6809 return BinaryOperator::CreateOr(Builder.CreateNot(B), A);
6810 }
6811}
6812
6813// Transform pattern like:
6814// (1 << Y) u<= X or ~(-1 << Y) u< X or ((1 << Y)+(-1)) u< X
6815// (1 << Y) u> X or ~(-1 << Y) u>= X or ((1 << Y)+(-1)) u>= X
6816// Into:
6817// (X l>> Y) != 0
6818// (X l>> Y) == 0
6820 InstCombiner::BuilderTy &Builder) {
6821 ICmpInst::Predicate Pred, NewPred;
6822 Value *X, *Y;
6823 if (match(&Cmp,
6824 m_c_ICmp(Pred, m_OneUse(m_Shl(m_One(), m_Value(Y))), m_Value(X)))) {
6825 switch (Pred) {
6826 case ICmpInst::ICMP_ULE:
6827 NewPred = ICmpInst::ICMP_NE;
6828 break;
6829 case ICmpInst::ICMP_UGT:
6830 NewPred = ICmpInst::ICMP_EQ;
6831 break;
6832 default:
6833 return nullptr;
6834 }
6835 } else if (match(&Cmp, m_c_ICmp(Pred,
6838 m_Add(m_Shl(m_One(), m_Value(Y)),
6839 m_AllOnes()))),
6840 m_Value(X)))) {
6841 // The variant with 'add' is not canonical, (the variant with 'not' is)
6842 // we only get it because it has extra uses, and can't be canonicalized,
6843
6844 switch (Pred) {
6845 case ICmpInst::ICMP_ULT:
6846 NewPred = ICmpInst::ICMP_NE;
6847 break;
6848 case ICmpInst::ICMP_UGE:
6849 NewPred = ICmpInst::ICMP_EQ;
6850 break;
6851 default:
6852 return nullptr;
6853 }
6854 } else
6855 return nullptr;
6856
6857 Value *NewX = Builder.CreateLShr(X, Y, X->getName() + ".highbits");
6858 Constant *Zero = Constant::getNullValue(NewX->getType());
6859 return CmpInst::Create(Instruction::ICmp, NewPred, NewX, Zero);
6860}
6861
6863 InstCombiner::BuilderTy &Builder) {
6864 const CmpInst::Predicate Pred = Cmp.getPredicate();
6865 Value *LHS = Cmp.getOperand(0), *RHS = Cmp.getOperand(1);
6866 Value *V1, *V2;
6867
6868 auto createCmpReverse = [&](CmpInst::Predicate Pred, Value *X, Value *Y) {
6869 Value *V = Builder.CreateCmp(Pred, X, Y, Cmp.getName());
6870 if (auto *I = dyn_cast<Instruction>(V))
6871 I->copyIRFlags(&Cmp);
6872 Module *M = Cmp.getModule();
6874 M, Intrinsic::experimental_vector_reverse, V->getType());
6875 return CallInst::Create(F, V);
6876 };
6877
6878 if (match(LHS, m_VecReverse(m_Value(V1)))) {
6879 // cmp Pred, rev(V1), rev(V2) --> rev(cmp Pred, V1, V2)
6880 if (match(RHS, m_VecReverse(m_Value(V2))) &&
6881 (LHS->hasOneUse() || RHS->hasOneUse()))
6882 return createCmpReverse(Pred, V1, V2);
6883
6884 // cmp Pred, rev(V1), RHSSplat --> rev(cmp Pred, V1, RHSSplat)
6885 if (LHS->hasOneUse() && isSplatValue(RHS))
6886 return createCmpReverse(Pred, V1, RHS);
6887 }
6888 // cmp Pred, LHSSplat, rev(V2) --> rev(cmp Pred, LHSSplat, V2)
6889 else if (isSplatValue(LHS) && match(RHS, m_OneUse(m_VecReverse(m_Value(V2)))))
6890 return createCmpReverse(Pred, LHS, V2);
6891
6892 ArrayRef<int> M;
6893 if (!match(LHS, m_Shuffle(m_Value(V1), m_Undef(), m_Mask(M))))
6894 return nullptr;
6895
6896 // If both arguments of the cmp are shuffles that use the same mask and
6897 // shuffle within a single vector, move the shuffle after the cmp:
6898 // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M
6899 Type *V1Ty = V1->getType();
6900 if (match(RHS, m_Shuffle(m_Value(V2), m_Undef(), m_SpecificMask(M))) &&
6901 V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) {
6902 Value *NewCmp = Builder.CreateCmp(Pred, V1, V2);
6903 return new ShuffleVectorInst(NewCmp, M);
6904 }
6905
6906 // Try to canonicalize compare with splatted operand and splat constant.
6907 // TODO: We could generalize this for more than splats. See/use the code in
6908 // InstCombiner::foldVectorBinop().
6909 Constant *C;
6910 if (!LHS->hasOneUse() || !match(RHS, m_Constant(C)))
6911 return nullptr;
6912
6913 // Length-changing splats are ok, so adjust the constants as needed:
6914 // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M
6915 Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true);
6916 int MaskSplatIndex;
6917 if (ScalarC && match(M, m_SplatOrUndefMask(MaskSplatIndex))) {
6918 // We allow undefs in matching, but this transform removes those for safety.
6919 // Demanded elements analysis should be able to recover some/all of that.
6920 C = ConstantVector::getSplat(cast<VectorType>(V1Ty)->getElementCount(),
6921 ScalarC);
6922 SmallVector<int, 8> NewM(M.size(), MaskSplatIndex);
6923 Value *NewCmp = Builder.CreateCmp(Pred, V1, C);
6924 return new ShuffleVectorInst(NewCmp, NewM);
6925 }
6926
6927 return nullptr;
6928}
6929
6930// extract(uadd.with.overflow(A, B), 0) ult A
6931// -> extract(uadd.with.overflow(A, B), 1)
6933 CmpInst::Predicate Pred = I.getPredicate();
6934 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
6935
6936 Value *UAddOv;
6937 Value *A, *B;
6938 auto UAddOvResultPat = m_ExtractValue<0>(
6939 m_Intrinsic<Intrinsic::uadd_with_overflow>(m_Value(A), m_Value(B)));
6940 if (match(Op0, UAddOvResultPat) &&
6941 ((Pred == ICmpInst::ICMP_ULT && (Op1 == A || Op1 == B)) ||
6942 (Pred == ICmpInst::ICMP_EQ && match(Op1, m_ZeroInt()) &&
6943 (match(A, m_One()) || match(B, m_One()))) ||
6944 (Pred == ICmpInst::ICMP_NE && match(Op1, m_AllOnes()) &&
6945 (match(A, m_AllOnes()) || match(B, m_AllOnes())))))
6946 // extract(uadd.with.overflow(A, B), 0) < A
6947 // extract(uadd.with.overflow(A, 1), 0) == 0
6948 // extract(uadd.with.overflow(A, -1), 0) != -1
6949 UAddOv = cast<ExtractValueInst>(Op0)->getAggregateOperand();
6950 else if (match(Op1, UAddOvResultPat) &&
6951 Pred == ICmpInst::ICMP_UGT && (Op0 == A || Op0 == B))
6952 // A > extract(uadd.with.overflow(A, B), 0)
6953 UAddOv = cast<ExtractValueInst>(Op1)->getAggregateOperand();
6954 else
6955 return nullptr;
6956
6957 return ExtractValueInst::Create(UAddOv, 1);
6958}
6959
6961 if (!I.getOperand(0)->getType()->isPointerTy() ||
6963 I.getParent()->getParent(),
6964 I.getOperand(0)->getType()->getPointerAddressSpace())) {
6965 return nullptr;
6966 }
6967 Instruction *Op;
6968 if (match(I.getOperand(0), m_Instruction(Op)) &&
6969 match(I.getOperand(1), m_Zero()) &&
6970 Op->isLaunderOrStripInvariantGroup()) {
6971 return ICmpInst::Create(Instruction::ICmp, I.getPredicate(),
6972 Op->getOperand(0), I.getOperand(1));
6973 }
6974 return nullptr;
6975}
6976
6977/// This function folds patterns produced by lowering of reduce idioms, such as
6978/// llvm.vector.reduce.and which are lowered into instruction chains. This code
6979/// attempts to generate fewer number of scalar comparisons instead of vector
6980/// comparisons when possible.
6982 InstCombiner::BuilderTy &Builder,
6983 const DataLayout &DL) {
6984 if (I.getType()->isVectorTy())
6985 return nullptr;
6986 ICmpInst::Predicate OuterPred, InnerPred;
6987 Value *LHS, *RHS;
6988
6989 // Match lowering of @llvm.vector.reduce.and. Turn
6990 /// %vec_ne = icmp ne <8 x i8> %lhs, %rhs
6991 /// %scalar_ne = bitcast <8 x i1> %vec_ne to i8
6992 /// %res = icmp <pred> i8 %scalar_ne, 0
6993 ///
6994 /// into
6995 ///
6996 /// %lhs.scalar = bitcast <8 x i8> %lhs to i64
6997 /// %rhs.scalar = bitcast <8 x i8> %rhs to i64
6998 /// %res = icmp <pred> i64 %lhs.scalar, %rhs.scalar
6999 ///
7000 /// for <pred> in {ne, eq}.
7001 if (!match(&I, m_ICmp(OuterPred,
7003 m_ICmp(InnerPred, m_Value(LHS), m_Value(RHS))))),
7004 m_Zero())))
7005 return nullptr;
7006 auto *LHSTy = dyn_cast<FixedVectorType>(LHS->getType());
7007 if (!LHSTy || !LHSTy->getElementType()->isIntegerTy())
7008 return nullptr;
7009 unsigned NumBits =
7010 LHSTy->getNumElements() * LHSTy->getElementType()->getIntegerBitWidth();
7011 // TODO: Relax this to "not wider than max legal integer type"?
7012 if (!DL.isLegalInteger(NumBits))
7013 return nullptr;
7014
7015 if (ICmpInst::isEquality(OuterPred) && InnerPred == ICmpInst::ICMP_NE) {
7016 auto *ScalarTy = Builder.getIntNTy(NumBits);
7017 LHS = Builder.CreateBitCast(LHS, ScalarTy, LHS->getName() + ".scalar");
7018 RHS = Builder.CreateBitCast(RHS, ScalarTy, RHS->getName() + ".scalar");
7019 return ICmpInst::Create(Instruction::ICmp, OuterPred, LHS, RHS,
7020 I.getName());
7021 }
7022
7023 return nullptr;
7024}
7025
7026// This helper will be called with icmp operands in both orders.
7028 Value *Op0, Value *Op1,
7029 ICmpInst &CxtI) {
7030 // Try to optimize 'icmp GEP, P' or 'icmp P, GEP'.
7031 if (auto *GEP = dyn_cast<GEPOperator>(Op0))
7032 if (Instruction *NI = foldGEPICmp(GEP, Op1, Pred, CxtI))
7033 return NI;
7034
7035 if (auto *SI = dyn_cast<SelectInst>(Op0))
7036 if (Instruction *NI = foldSelectICmp(Pred, SI, Op1, CxtI))
7037 return NI;
7038
7039 if (auto *MinMax = dyn_cast<MinMaxIntrinsic>(Op0))
7040 if (Instruction *Res = foldICmpWithMinMax(CxtI, MinMax, Op1, Pred))
7041 return Res;
7042
7043 {
7044 Value *X;
7045 const APInt *C;
7046 // icmp X+Cst, X
7047 if (match(Op0, m_Add(m_Value(X), m_APInt(C))) && Op1 == X)
7048 return foldICmpAddOpConst(X, *C, Pred);
7049 }
7050
7051 // abs(X) >= X --> true
7052 // abs(X) u<= X --> true
7053 // abs(X) < X --> false
7054 // abs(X) u> X --> false
7055 // abs(X) u>= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
7056 // abs(X) <= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
7057 // abs(X) == X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
7058 // abs(X) u< X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
7059 // abs(X) > X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
7060 // abs(X) != X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
7061 {
7062 Value *X;
7063 Constant *C;
7064 if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(X), m_Constant(C))) &&
7065 match(Op1, m_Specific(X))) {
7066 Value *NullValue = Constant::getNullValue(X->getType());
7067 Value *AllOnesValue = Constant::getAllOnesValue(X->getType());
7068 const APInt SMin =
7069 APInt::getSignedMinValue(X->getType()->getScalarSizeInBits());
7070 bool IsIntMinPosion = C->isAllOnesValue();
7071 switch (Pred) {
7072 case CmpInst::ICMP_ULE:
7073 case CmpInst::ICMP_SGE:
7074 return replaceInstUsesWith(CxtI, ConstantInt::getTrue(CxtI.getType()));
7075 case CmpInst::ICMP_UGT:
7076 case CmpInst::ICMP_SLT:
7078 case CmpInst::ICMP_UGE:
7079 case CmpInst::ICMP_SLE:
7080 case CmpInst::ICMP_EQ: {
7081 return replaceInstUsesWith(
7082 CxtI, IsIntMinPosion
7083 ? Builder.CreateICmpSGT(X, AllOnesValue)
7085 X, ConstantInt::get(X->getType(), SMin + 1)));
7086 }
7087 case CmpInst::ICMP_ULT:
7088 case CmpInst::ICMP_SGT:
7089 case CmpInst::ICMP_NE: {
7090 return replaceInstUsesWith(
7091 CxtI, IsIntMinPosion
7092 ? Builder.CreateICmpSLT(X, NullValue)
7094 X, ConstantInt::get(X->getType(), SMin)));
7095 }
7096 default:
7097 llvm_unreachable("Invalid predicate!");
7098 }
7099 }
7100 }
7101
7102 const SimplifyQuery Q = SQ.getWithInstruction(&CxtI);
7103 if (Value *V = foldICmpWithLowBitMaskedVal(Pred, Op0, Op1, Q, *this))
7104 return replaceInstUsesWith(CxtI, V);
7105
7106 return nullptr;
7107}
7108
7110 bool Changed = false;
7112 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
7113 unsigned Op0Cplxity = getComplexity(Op0);
7114 unsigned Op1Cplxity = getComplexity(Op1);
7115
7116 /// Orders the operands of the compare so that they are listed from most
7117 /// complex to least complex. This puts constants before unary operators,
7118 /// before binary operators.
7119 if (Op0Cplxity < Op1Cplxity) {
7120 I.swapOperands();
7121 std::swap(Op0, Op1);
7122 Changed = true;
7123 }
7124
7125 if (Value *V = simplifyICmpInst(I.getPredicate(), Op0, Op1, Q))
7126 return replaceInstUsesWith(I, V);
7127
7128 // Comparing -val or val with non-zero is the same as just comparing val
7129 // ie, abs(val) != 0 -> val != 0
7130 if (I.getPredicate() == ICmpInst::ICMP_NE && match(Op1, m_Zero())) {
7131 Value *Cond, *SelectTrue, *SelectFalse;
7132 if (match(Op0, m_Select(m_Value(Cond), m_Value(SelectTrue),
7133 m_Value(SelectFalse)))) {
7134 if (Value *V = dyn_castNegVal(SelectTrue)) {
7135 if (V == SelectFalse)
7136 return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1);
7137 }
7138 else if (Value *V = dyn_castNegVal(SelectFalse)) {
7139 if (V == SelectTrue)
7140 return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1);
7141 }
7142 }
7143 }
7144
7145 if (Op0->getType()->isIntOrIntVectorTy(1))
7147 return Res;
7148
7150 return Res;
7151
7153 return Res;
7154
7156 return Res;
7157
7159 return Res;
7160
7162 return Res;
7163
7165 return Res;
7166
7168 return Res;
7169
7170 // Test if the ICmpInst instruction is used exclusively by a select as
7171 // part of a minimum or maximum operation. If so, refrain from doing
7172 // any other folding. This helps out other analyses which understand
7173 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
7174 // and CodeGen. And in this case, at least one of the comparison
7175 // operands has at least one user besides the compare (the select),
7176 // which would often largely negate the benefit of folding anyway.
7177 //
7178 // Do the same for the other patterns recognized by matchSelectPattern.
7179 if (I.hasOneUse())
7180 if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) {
7181 Value *A, *B;
7183 if (SPR.Flavor != SPF_UNKNOWN)
7184 return nullptr;
7185 }
7186
7187 // Do this after checking for min/max to prevent infinite looping.
7188 if (Instruction *Res = foldICmpWithZero(I))
7189 return Res;
7190
7191 // FIXME: We only do this after checking for min/max to prevent infinite
7192 // looping caused by a reverse canonicalization of these patterns for min/max.
7193 // FIXME: The organization of folds is a mess. These would naturally go into
7194 // canonicalizeCmpWithConstant(), but we can't move all of the above folds
7195 // down here after the min/max restriction.
7196 ICmpInst::Predicate Pred = I.getPredicate();
7197 const APInt *C;
7198 if (match(Op1, m_APInt(C))) {
7199 // For i32: x >u 2147483647 -> x <s 0 -> true if sign bit set
7200 if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) {
7201 Constant *Zero = Constant::getNullValue(Op0->getType());
7202 return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero);
7203 }
7204
7205 // For i32: x <u 2147483648 -> x >s -1 -> true if sign bit clear
7206 if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) {
7208 return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes);
7209 }
7210 }
7211
7212 // The folds in here may rely on wrapping flags and special constants, so
7213 // they can break up min/max idioms in some cases but not seemingly similar
7214 // patterns.
7215 // FIXME: It may be possible to enhance select folding to make this
7216 // unnecessary. It may also be moot if we canonicalize to min/max
7217 // intrinsics.
7218 if (Instruction *Res = foldICmpBinOp(I, Q))
7219 return Res;
7220
7222 return Res;
7223
7224 // Try to match comparison as a sign bit test. Intentionally do this after
7225 // foldICmpInstWithConstant() to potentially let other folds to happen first.
7226 if (Instruction *New = foldSignBitTest(I))
7227 return New;
7228
7230 return Res;
7231
7232 if (Instruction *Res = foldICmpCommutative(I.getPredicate(), Op0, Op1, I))
7233 return Res;
7234 if (Instruction *Res =
7235 foldICmpCommutative(I.getSwappedPredicate(), Op1, Op0, I))
7236 return Res;
7237
7238 if (I.isCommutative()) {
7239 if (auto Pair = matchSymmetricPair(I.getOperand(0), I.getOperand(1))) {
7240 replaceOperand(I, 0, Pair->first);
7241 replaceOperand(I, 1, Pair->second);
7242 return &I;
7243 }
7244 }
7245
7246 // In case of a comparison with two select instructions having the same
7247 // condition, check whether one of the resulting branches can be simplified.
7248 // If so, just compare the other branch and select the appropriate result.
7249 // For example:
7250 // %tmp1 = select i1 %cmp, i32 %y, i32 %x
7251 // %tmp2 = select i1 %cmp, i32 %z, i32 %x
7252 // %cmp2 = icmp slt i32 %tmp2, %tmp1
7253 // The icmp will result false for the false value of selects and the result
7254 // will depend upon the comparison of true values of selects if %cmp is
7255 // true. Thus, transform this into:
7256 // %cmp = icmp slt i32 %y, %z
7257 // %sel = select i1 %cond, i1 %cmp, i1 false
7258 // This handles similar cases to transform.
7259 {
7260 Value *Cond, *A, *B, *C, *D;
7261 if (match(Op0, m_Select(m_Value(Cond), m_Value(A), m_Value(B))) &&
7263 (Op0->hasOneUse() || Op1->hasOneUse())) {
7264 // Check whether comparison of TrueValues can be simplified
7265 if (Value *Res = simplifyICmpInst(Pred, A, C, SQ)) {
7266 Value *NewICMP = Builder.CreateICmp(Pred, B, D);
7267 return SelectInst::Create(Cond, Res, NewICMP);
7268 }
7269 // Check whether comparison of FalseValues can be simplified
7270 if (Value *Res = simplifyICmpInst(Pred, B, D, SQ)) {
7271 Value *NewICMP = Builder.CreateICmp(Pred, A, C);
7272 return SelectInst::Create(Cond, NewICMP, Res);
7273 }
7274 }
7275 }
7276
7277 // Try to optimize equality comparisons against alloca-based pointers.
7278 if (Op0->getType()->isPointerTy() && I.isEquality()) {
7279 assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?");
7280 if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0)))
7281 if (foldAllocaCmp(Alloca))
7282 return nullptr;
7283 if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1)))
7284 if (foldAllocaCmp(Alloca))
7285 return nullptr;
7286 }
7287
7288 if (Instruction *Res = foldICmpBitCast(I))
7289 return Res;
7290
7291 // TODO: Hoist this above the min/max bailout.
7293 return R;
7294
7295 {
7296 Value *X, *Y;
7297 // Transform (X & ~Y) == 0 --> (X & Y) != 0
7298 // and (X & ~Y) != 0 --> (X & Y) == 0
7299 // if A is a power of 2.
7300 if (match(Op0, m_And(m_Value(X), m_Not(m_Value(Y)))) &&
7301 match(Op1, m_Zero()) && isKnownToBeAPowerOfTwo(X, false, 0, &I) &&
7302 I.isEquality())
7303 return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(X, Y),
7304 Op1);
7305
7306 // Op0 pred Op1 -> ~Op1 pred ~Op0, if this allows us to drop an instruction.
7307 if (Op0->getType()->isIntOrIntVectorTy()) {
7308 bool ConsumesOp0, ConsumesOp1;
7309 if (isFreeToInvert(Op0, Op0->hasOneUse(), ConsumesOp0) &&
7310 isFreeToInvert(Op1, Op1->hasOneUse(), ConsumesOp1) &&
7311 (ConsumesOp0 || ConsumesOp1)) {
7312 Value *InvOp0 = getFreelyInverted(Op0, Op0->hasOneUse(), &Builder);
7313 Value *InvOp1 = getFreelyInverted(Op1, Op1->hasOneUse(), &Builder);
7314 assert(InvOp0 && InvOp1 &&
7315 "Mismatch between isFreeToInvert and getFreelyInverted");
7316 return new ICmpInst(I.getSwappedPredicate(), InvOp0, InvOp1);
7317 }
7318 }
7319
7320 Instruction *AddI = nullptr;
7322 m_Instruction(AddI))) &&
7323 isa<IntegerType>(X->getType())) {
7324 Value *Result;
7325 Constant *Overflow;
7326 // m_UAddWithOverflow can match patterns that do not include an explicit
7327 // "add" instruction, so check the opcode of the matched op.
7328 if (AddI->getOpcode() == Instruction::Add &&
7329 OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, X, Y, *AddI,
7330 Result, Overflow)) {
7331 replaceInstUsesWith(*AddI, Result);
7332 eraseInstFromFunction(*AddI);
7333 return replaceInstUsesWith(I, Overflow);
7334 }
7335 }
7336
7337 // (zext X) * (zext Y) --> llvm.umul.with.overflow.
7338 if (match(Op0, m_NUWMul(m_ZExt(m_Value(X)), m_ZExt(m_Value(Y)))) &&
7339 match(Op1, m_APInt(C))) {
7340 if (Instruction *R = processUMulZExtIdiom(I, Op0, C, *this))
7341 return R;
7342 }
7343
7344 // Signbit test folds
7345 // Fold (X u>> BitWidth - 1 Pred ZExt(i1)) --> X s< 0 Pred i1
7346 // Fold (X s>> BitWidth - 1 Pred SExt(i1)) --> X s< 0 Pred i1
7347 Instruction *ExtI;
7348 if ((I.isUnsigned() || I.isEquality()) &&
7349 match(Op1,
7351 Y->getType()->getScalarSizeInBits() == 1 &&
7352 (Op0->hasOneUse() || Op1->hasOneUse())) {
7353 unsigned OpWidth = Op0->getType()->getScalarSizeInBits();
7354 Instruction *ShiftI;
7355 if (match(Op0, m_CombineAnd(m_Instruction(ShiftI),
7357 OpWidth - 1))))) {
7358 unsigned ExtOpc = ExtI->getOpcode();
7359 unsigned ShiftOpc = ShiftI->getOpcode();
7360 if ((ExtOpc == Instruction::ZExt && ShiftOpc == Instruction::LShr) ||
7361 (ExtOpc == Instruction::SExt && ShiftOpc == Instruction::AShr)) {
7362 Value *SLTZero =
7364 Value *Cmp = Builder.CreateICmp(Pred, SLTZero, Y, I.getName());
7365 return replaceInstUsesWith(I, Cmp);
7366 }
7367 }
7368 }
7369 }
7370
7371 if (Instruction *Res = foldICmpEquality(I))
7372 return Res;
7373
7375 return Res;
7376
7377 if (Instruction *Res = foldICmpOfUAddOv(I))
7378 return Res;
7379
7380 // The 'cmpxchg' instruction returns an aggregate containing the old value and
7381 // an i1 which indicates whether or not we successfully did the swap.
7382 //
7383 // Replace comparisons between the old value and the expected value with the
7384 // indicator that 'cmpxchg' returns.
7385 //
7386 // N.B. This transform is only valid when the 'cmpxchg' is not permitted to
7387 // spuriously fail. In those cases, the old value may equal the expected
7388 // value but it is possible for the swap to not occur.
7389 if (I.getPredicate() == ICmpInst::ICMP_EQ)
7390 if (auto *EVI = dyn_cast<ExtractValueInst>(Op0))
7391 if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(EVI->getAggregateOperand()))
7392 if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 &&
7393 !ACXI->isWeak())
7394 return ExtractValueInst::Create(ACXI, 1);
7395
7397 return Res;
7398
7399 if (I.getType()->isVectorTy())
7400 if (Instruction *Res = foldVectorCmp(I, Builder))
7401 return Res;
7402
7404 return Res;
7405
7407 return Res;
7408
7409 return Changed ? &I : nullptr;
7410}
7411
7412/// Fold fcmp ([us]itofp x, cst) if possible.
7414 Instruction *LHSI,
7415 Constant *RHSC) {
7416 const APFloat *RHS;
7417 if (!match(RHSC, m_APFloat(RHS)))
7418 return nullptr;
7419
7420 // Get the width of the mantissa. We don't want to hack on conversions that
7421 // might lose information from the integer, e.g. "i64 -> float"
7422 int MantissaWidth = LHSI->getType()->getFPMantissaWidth();
7423 if (MantissaWidth == -1) return nullptr; // Unknown.
7424
7425 Type *IntTy = LHSI->getOperand(0)->getType();
7426 unsigned IntWidth = IntTy->getScalarSizeInBits();
7427 bool LHSUnsigned = isa<UIToFPInst>(LHSI);
7428
7429 if (I.isEquality()) {
7430 FCmpInst::Predicate P = I.getPredicate();
7431 bool IsExact = false;
7432 APSInt RHSCvt(IntWidth, LHSUnsigned);
7433 RHS->convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact);
7434
7435 // If the floating point constant isn't an integer value, we know if we will
7436 // ever compare equal / not equal to it.
7437 if (!IsExact) {
7438 // TODO: Can never be -0.0 and other non-representable values
7439 APFloat RHSRoundInt(*RHS);
7441 if (*RHS != RHSRoundInt) {
7443 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7444
7446 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7447 }
7448 }
7449
7450 // TODO: If the constant is exactly representable, is it always OK to do
7451 // equality compares as integer?
7452 }
7453
7454 // Check to see that the input is converted from an integer type that is small
7455 // enough that preserves all bits. TODO: check here for "known" sign bits.
7456 // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e.
7457
7458 // Following test does NOT adjust IntWidth downwards for signed inputs,
7459 // because the most negative value still requires all the mantissa bits
7460 // to distinguish it from one less than that value.
7461 if ((int)IntWidth > MantissaWidth) {
7462 // Conversion would lose accuracy. Check if loss can impact comparison.
7463 int Exp = ilogb(*RHS);
7464 if (Exp == APFloat::IEK_Inf) {
7465 int MaxExponent = ilogb(APFloat::getLargest(RHS->getSemantics()));
7466 if (MaxExponent < (int)IntWidth - !LHSUnsigned)
7467 // Conversion could create infinity.
7468 return nullptr;
7469 } else {
7470 // Note that if RHS is zero or NaN, then Exp is negative
7471 // and first condition is trivially false.
7472 if (MantissaWidth <= Exp && Exp <= (int)IntWidth - !LHSUnsigned)
7473 // Conversion could affect comparison.
7474 return nullptr;
7475 }
7476 }
7477
7478 // Otherwise, we can potentially simplify the comparison. We know that it
7479 // will always come through as an integer value and we know the constant is
7480 // not a NAN (it would have been previously simplified).
7481 assert(!RHS->isNaN() && "NaN comparison not already folded!");
7482
7484 switch (I.getPredicate()) {
7485 default: llvm_unreachable("Unexpected predicate!");
7486 case FCmpInst::FCMP_UEQ:
7487 case FCmpInst::FCMP_OEQ:
7488 Pred = ICmpInst::ICMP_EQ;
7489 break;
7490 case FCmpInst::FCMP_UGT:
7491 case FCmpInst::FCMP_OGT:
7492 Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT;
7493 break;
7494 case FCmpInst::FCMP_UGE:
7495 case FCmpInst::FCMP_OGE:
7496 Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE;
7497 break;
7498 case FCmpInst::FCMP_ULT:
7499 case FCmpInst::FCMP_OLT:
7500 Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT;
7501 break;
7502 case FCmpInst::FCMP_ULE:
7503 case FCmpInst::FCMP_OLE:
7504 Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE;
7505 break;
7506 case FCmpInst::FCMP_UNE:
7507 case FCmpInst::FCMP_ONE:
7508 Pred = ICmpInst::ICMP_NE;
7509 break;
7510 case FCmpInst::FCMP_ORD:
7511 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7512 case FCmpInst::FCMP_UNO:
7513 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7514 }
7515
7516 // Now we know that the APFloat is a normal number, zero or inf.
7517
7518 // See if the FP constant is too large for the integer. For example,
7519 // comparing an i8 to 300.0.
7520 if (!LHSUnsigned) {
7521 // If the RHS value is > SignedMax, fold the comparison. This handles +INF
7522 // and large values.
7523 APFloat SMax(RHS->getSemantics());
7524 SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true,
7526 if (SMax < *RHS) { // smax < 13123.0
7527 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT ||
7528 Pred == ICmpInst::ICMP_SLE)
7529 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7530 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7531 }
7532 } else {
7533 // If the RHS value is > UnsignedMax, fold the comparison. This handles
7534 // +INF and large values.
7535 APFloat UMax(RHS->getSemantics());
7536 UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false,
7538 if (UMax < *RHS) { // umax < 13123.0
7539 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT ||
7540 Pred == ICmpInst::ICMP_ULE)
7541 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7542 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7543 }
7544 }
7545
7546 if (!LHSUnsigned) {
7547 // See if the RHS value is < SignedMin.
7548 APFloat SMin(RHS->getSemantics());
7549 SMin.convertFromAPInt(APInt::getSignedMinValue(IntWidth), true,
7551 if (SMin > *RHS) { // smin > 12312.0
7552 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT ||
7553 Pred == ICmpInst::ICMP_SGE)
7554 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7555 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7556 }
7557 } else {
7558 // See if the RHS value is < UnsignedMin.
7559 APFloat UMin(RHS->getSemantics());
7560 UMin.convertFromAPInt(APInt::getMinValue(IntWidth), false,
7562 if (UMin > *RHS) { // umin > 12312.0
7563 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT ||
7564 Pred == ICmpInst::ICMP_UGE)
7565 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7566 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7567 }
7568 }
7569
7570 // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or
7571 // [0, UMAX], but it may still be fractional. Check whether this is the case
7572 // using the IsExact flag.
7573 // Don't do this for zero, because -0.0 is not fractional.
7574 APSInt RHSInt(IntWidth, LHSUnsigned);
7575 bool IsExact;
7576 RHS->convertToInteger(RHSInt, APFloat::rmTowardZero, &IsExact);
7577 if (!RHS->isZero()) {
7578 if (!IsExact) {
7579 // If we had a comparison against a fractional value, we have to adjust
7580 // the compare predicate and sometimes the value. RHSC is rounded towards
7581 // zero at this point.
7582 switch (Pred) {
7583 default: llvm_unreachable("Unexpected integer comparison!");
7584 case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true
7585 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7586 case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false
7587 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7588 case ICmpInst::ICMP_ULE:
7589 // (float)int <= 4.4 --> int <= 4
7590 // (float)int <= -4.4 --> false
7591 if (RHS->isNegative())
7592 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7593 break;
7594 case ICmpInst::ICMP_SLE:
7595 // (float)int <= 4.4 --> int <= 4
7596 // (float)int <= -4.4 --> int < -4
7597 if (RHS->isNegative())
7598 Pred = ICmpInst::ICMP_SLT;
7599 break;
7600 case ICmpInst::ICMP_ULT:
7601 // (float)int < -4.4 --> false
7602 // (float)int < 4.4 --> int <= 4
7603 if (RHS->isNegative())
7604 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7605 Pred = ICmpInst::ICMP_ULE;
7606 break;
7607 case ICmpInst::ICMP_SLT:
7608 // (float)int < -4.4 --> int < -4
7609 // (float)int < 4.4 --> int <= 4
7610 if (!RHS->isNegative())
7611 Pred = ICmpInst::ICMP_SLE;
7612 break;
7613 case ICmpInst::ICMP_UGT:
7614 // (float)int > 4.4 --> int > 4
7615 // (float)int > -4.4 --> true
7616 if (RHS->isNegative())
7617 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7618 break;
7619 case ICmpInst::ICMP_SGT:
7620 // (float)int > 4.4 --> int > 4
7621 // (float)int > -4.4 --> int >= -4
7622 if (RHS->isNegative())
7623 Pred = ICmpInst::ICMP_SGE;
7624 break;
7625 case ICmpInst::ICMP_UGE:
7626 // (float)int >= -4.4 --> true
7627 // (float)int >= 4.4 --> int > 4
7628 if (RHS->isNegative())
7629 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7630 Pred = ICmpInst::ICMP_UGT;
7631 break;
7632 case ICmpInst::ICMP_SGE:
7633 // (float)int >= -4.4 --> int >= -4
7634 // (float)int >= 4.4 --> int > 4
7635 if (!RHS->isNegative())
7636 Pred = ICmpInst::ICMP_SGT;
7637 break;
7638 }
7639 }
7640 }
7641
7642 // Lower this FP comparison into an appropriate integer version of the
7643 // comparison.
7644 return new ICmpInst(Pred, LHSI->getOperand(0),
7645 ConstantInt::get(LHSI->getOperand(0)->getType(), RHSInt));
7646}
7647
7648/// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary.
7650 Constant *RHSC) {
7651 // When C is not 0.0 and infinities are not allowed:
7652 // (C / X) < 0.0 is a sign-bit test of X
7653 // (C / X) < 0.0 --> X < 0.0 (if C is positive)
7654 // (C / X) < 0.0 --> X > 0.0 (if C is negative, swap the predicate)
7655 //
7656 // Proof:
7657 // Multiply (C / X) < 0.0 by X * X / C.
7658 // - X is non zero, if it is the flag 'ninf' is violated.
7659 // - C defines the sign of X * X * C. Thus it also defines whether to swap
7660 // the predicate. C is also non zero by definition.
7661 //
7662 // Thus X * X / C is non zero and the transformation is valid. [qed]
7663
7664 FCmpInst::Predicate Pred = I.getPredicate();
7665
7666 // Check that predicates are valid.
7667 if ((Pred != FCmpInst::FCMP_OGT) && (Pred != FCmpInst::FCMP_OLT) &&
7668 (Pred != FCmpInst::FCMP_OGE) && (Pred != FCmpInst::FCMP_OLE))
7669 return nullptr;
7670
7671 // Check that RHS operand is zero.
7672 if (!match(RHSC, m_AnyZeroFP()))
7673 return nullptr;
7674
7675 // Check fastmath flags ('ninf').
7676 if (!LHSI->hasNoInfs() || !I.hasNoInfs())
7677 return nullptr;
7678
7679 // Check the properties of the dividend. It must not be zero to avoid a
7680 // division by zero (see Proof).
7681 const APFloat *C;
7682 if (!match(LHSI->getOperand(0), m_APFloat(C)))
7683 return nullptr;
7684
7685 if (C->isZero())
7686 return nullptr;
7687
7688 // Get swapped predicate if necessary.
7689 if (C->isNegative())
7690 Pred = I.getSwappedPredicate();
7691
7692 return new FCmpInst(Pred, LHSI->getOperand(1), RHSC, "", &I);
7693}
7694
7695/// Optimize fabs(X) compared with zero.
7697 Value *X;
7698 if (!match(I.getOperand(0), m_FAbs(m_Value(X))))
7699 return nullptr;
7700
7701 const APFloat *C;
7702 if (!match(I.getOperand(1), m_APFloat(C)))
7703 return nullptr;
7704
7705 if (!C->isPosZero()) {
7706 if (!C->isSmallestNormalized())
7707 return nullptr;
7708
7709 const Function *F = I.getFunction();
7710 DenormalMode Mode = F->getDenormalMode(C->getSemantics());
7711 if (Mode.Input == DenormalMode::PreserveSign ||
7712 Mode.Input == DenormalMode::PositiveZero) {
7713
7714 auto replaceFCmp = [](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
7715 Constant *Zero = ConstantFP::getZero(X->getType());
7716 return new FCmpInst(P, X, Zero, "", I);
7717 };
7718
7719 switch (I.getPredicate()) {
7720 case FCmpInst::FCMP_OLT:
7721 // fcmp olt fabs(x), smallest_normalized_number -> fcmp oeq x, 0.0
7722 return replaceFCmp(&I, FCmpInst::FCMP_OEQ, X);
7723 case FCmpInst::FCMP_UGE:
7724 // fcmp uge fabs(x), smallest_normalized_number -> fcmp une x, 0.0
7725 return replaceFCmp(&I, FCmpInst::FCMP_UNE, X);
7726 case FCmpInst::FCMP_OGE:
7727 // fcmp oge fabs(x), smallest_normalized_number -> fcmp one x, 0.0
7728 return replaceFCmp(&I, FCmpInst::FCMP_ONE, X);
7729 case FCmpInst::FCMP_ULT:
7730 // fcmp ult fabs(x), smallest_normalized_number -> fcmp ueq x, 0.0
7731 return replaceFCmp(&I, FCmpInst::FCMP_UEQ, X);
7732 default:
7733 break;
7734 }
7735 }
7736
7737 return nullptr;
7738 }
7739
7740 auto replacePredAndOp0 = [&IC](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
7741 I->setPredicate(P);
7742 return IC.replaceOperand(*I, 0, X);
7743 };
7744
7745 switch (I.getPredicate()) {
7746 case FCmpInst::FCMP_UGE:
7747 case FCmpInst::FCMP_OLT:
7748 // fabs(X) >= 0.0 --> true
7749 // fabs(X) < 0.0 --> false
7750 llvm_unreachable("fcmp should have simplified");
7751
7752 case FCmpInst::FCMP_OGT:
7753 // fabs(X) > 0.0 --> X != 0.0
7754 return replacePredAndOp0(&I, FCmpInst::FCMP_ONE, X);
7755
7756 case FCmpInst::FCMP_UGT:
7757 // fabs(X) u> 0.0 --> X u!= 0.0
7758 return replacePredAndOp0(&I, FCmpInst::FCMP_UNE, X);
7759
7760 case FCmpInst::FCMP_OLE:
7761 // fabs(X) <= 0.0 --> X == 0.0
7762 return replacePredAndOp0(&I, FCmpInst::FCMP_OEQ, X);
7763
7764 case FCmpInst::FCMP_ULE:
7765 // fabs(X) u<= 0.0 --> X u== 0.0
7766 return replacePredAndOp0(&I, FCmpInst::FCMP_UEQ, X);
7767
7768 case FCmpInst::FCMP_OGE:
7769 // fabs(X) >= 0.0 --> !isnan(X)
7770 assert(!I.hasNoNaNs() && "fcmp should have simplified");
7771 return replacePredAndOp0(&I, FCmpInst::FCMP_ORD, X);
7772
7773 case FCmpInst::FCMP_ULT:
7774 // fabs(X) u< 0.0 --> isnan(X)
7775 assert(!I.hasNoNaNs() && "fcmp should have simplified");
7776 return replacePredAndOp0(&I, FCmpInst::FCMP_UNO, X);
7777
7778 case FCmpInst::FCMP_OEQ:
7779 case FCmpInst::FCMP_UEQ:
7780 case FCmpInst::FCMP_ONE:
7781 case FCmpInst::FCMP_UNE:
7782 case FCmpInst::FCMP_ORD:
7783 case FCmpInst::FCMP_UNO:
7784 // Look through the fabs() because it doesn't change anything but the sign.
7785 // fabs(X) == 0.0 --> X == 0.0,
7786 // fabs(X) != 0.0 --> X != 0.0
7787 // isnan(fabs(X)) --> isnan(X)
7788 // !isnan(fabs(X) --> !isnan(X)
7789 return replacePredAndOp0(&I, I.getPredicate(), X);
7790
7791 default:
7792 return nullptr;
7793 }
7794}
7795
7797 CmpInst::Predicate Pred = I.getPredicate();
7798 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
7799
7800 // Canonicalize fneg as Op1.
7801 if (match(Op0, m_FNeg(m_Value())) && !match(Op1, m_FNeg(m_Value()))) {
7802 std::swap(Op0, Op1);
7803 Pred = I.getSwappedPredicate();
7804 }
7805
7806 if (!match(Op1, m_FNeg(m_Specific(Op0))))
7807 return nullptr;
7808
7809 // Replace the negated operand with 0.0:
7810 // fcmp Pred Op0, -Op0 --> fcmp Pred Op0, 0.0
7811 Constant *Zero = ConstantFP::getZero(Op0->getType());
7812 return new FCmpInst(Pred, Op0, Zero, "", &I);
7813}
7814
7816 bool Changed = false;
7817
7818 /// Orders the operands of the compare so that they are listed from most
7819 /// complex to least complex. This puts constants before unary operators,
7820 /// before binary operators.
7821 if (getComplexity(I.getOperand(0)) < getComplexity(I.getOperand(1))) {
7822 I.swapOperands();
7823 Changed = true;
7824 }
7825
7826 const CmpInst::Predicate Pred = I.getPredicate();
7827 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
7828 if (Value *V = simplifyFCmpInst(Pred, Op0, Op1, I.getFastMathFlags(),
7830 return replaceInstUsesWith(I, V);
7831
7832 // Simplify 'fcmp pred X, X'
7833 Type *OpType = Op0->getType();
7834 assert(OpType == Op1->getType() && "fcmp with different-typed operands?");
7835 if (Op0 == Op1) {
7836 switch (Pred) {
7837 default: break;
7838 case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y)
7839 case FCmpInst::FCMP_ULT: // True if unordered or less than
7840 case FCmpInst::FCMP_UGT: // True if unordered or greater than
7841 case FCmpInst::FCMP_UNE: // True if unordered or not equal
7842 // Canonicalize these to be 'fcmp uno %X, 0.0'.
7843 I.setPredicate(FCmpInst::FCMP_UNO);
7844 I.setOperand(1, Constant::getNullValue(OpType));
7845 return &I;
7846
7847 case FCmpInst::FCMP_ORD: // True if ordered (no nans)
7848 case FCmpInst::FCMP_OEQ: // True if ordered and equal
7849 case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal
7850 case FCmpInst::FCMP_OLE: // True if ordered and less than or equal
7851 // Canonicalize these to be 'fcmp ord %X, 0.0'.
7852 I.setPredicate(FCmpInst::FCMP_ORD);
7853 I.setOperand(1, Constant::getNullValue(OpType));
7854 return &I;
7855 }
7856 }
7857
7858 if (I.isCommutative()) {
7859 if (auto Pair = matchSymmetricPair(I.getOperand(0), I.getOperand(1))) {
7860 replaceOperand(I, 0, Pair->first);
7861 replaceOperand(I, 1, Pair->second);
7862 return &I;
7863 }
7864 }
7865
7866 // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand,
7867 // then canonicalize the operand to 0.0.
7868 if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
7869 if (!match(Op0, m_PosZeroFP()) &&
7870 isKnownNeverNaN(Op0, 0, getSimplifyQuery().getWithInstruction(&I)))
7871 return replaceOperand(I, 0, ConstantFP::getZero(OpType));
7872
7873 if (!match(Op1, m_PosZeroFP()) &&
7874 isKnownNeverNaN(Op1, 0, getSimplifyQuery().getWithInstruction(&I)))
7875 return replaceOperand(I, 1, ConstantFP::getZero(OpType));
7876 }
7877
7878 // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y
7879 Value *X, *Y;
7880 if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y))))
7881 return new FCmpInst(I.getSwappedPredicate(), X, Y, "", &I);
7882
7884 return R;
7885
7886 // Test if the FCmpInst instruction is used exclusively by a select as
7887 // part of a minimum or maximum operation. If so, refrain from doing
7888 // any other folding. This helps out other analyses which understand
7889 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
7890 // and CodeGen. And in this case, at least one of the comparison
7891 // operands has at least one user besides the compare (the select),
7892 // which would often largely negate the benefit of folding anyway.
7893 if (I.hasOneUse())
7894 if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) {
7895 Value *A, *B;
7897 if (SPR.Flavor != SPF_UNKNOWN)
7898 return nullptr;
7899 }
7900
7901 // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0:
7902 // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0
7903 if (match(Op1, m_AnyZeroFP()) && !match(Op1, m_PosZeroFP()))
7904 return replaceOperand(I, 1, ConstantFP::getZero(OpType));
7905
7906 // Canonicalize:
7907 // fcmp olt X, +inf -> fcmp one X, +inf
7908 // fcmp ole X, +inf -> fcmp ord X, 0
7909 // fcmp ogt X, +inf -> false
7910 // fcmp oge X, +inf -> fcmp oeq X, +inf
7911 // fcmp ult X, +inf -> fcmp une X, +inf
7912 // fcmp ule X, +inf -> true
7913 // fcmp ugt X, +inf -> fcmp uno X, 0
7914 // fcmp uge X, +inf -> fcmp ueq X, +inf
7915 // fcmp olt X, -inf -> false
7916 // fcmp ole X, -inf -> fcmp oeq X, -inf
7917 // fcmp ogt X, -inf -> fcmp one X, -inf
7918 // fcmp oge X, -inf -> fcmp ord X, 0
7919 // fcmp ult X, -inf -> fcmp uno X, 0
7920 // fcmp ule X, -inf -> fcmp ueq X, -inf
7921 // fcmp ugt X, -inf -> fcmp une X, -inf
7922 // fcmp uge X, -inf -> true
7923 const APFloat *C;
7924 if (match(Op1, m_APFloat(C)) && C->isInfinity()) {
7925 switch (C->isNegative() ? FCmpInst::getSwappedPredicate(Pred) : Pred) {
7926 default:
7927 break;
7928 case FCmpInst::FCMP_ORD:
7929 case FCmpInst::FCMP_UNO:
7932 case FCmpInst::FCMP_OGT:
7933 case FCmpInst::FCMP_ULE:
7934 llvm_unreachable("Should be simplified by InstSimplify");
7935 case FCmpInst::FCMP_OLT:
7936 return new FCmpInst(FCmpInst::FCMP_ONE, Op0, Op1, "", &I);
7937 case FCmpInst::FCMP_OLE:
7938 return new FCmpInst(FCmpInst::FCMP_ORD, Op0, ConstantFP::getZero(OpType),
7939 "", &I);
7940 case FCmpInst::FCMP_OGE:
7941 return new FCmpInst(FCmpInst::FCMP_OEQ, Op0, Op1, "", &I);
7942 case FCmpInst::FCMP_ULT:
7943 return new FCmpInst(FCmpInst::FCMP_UNE, Op0, Op1, "", &I);
7944 case FCmpInst::FCMP_UGT:
7945 return new FCmpInst(FCmpInst::FCMP_UNO, Op0, ConstantFP::getZero(OpType),
7946 "", &I);
7947 case FCmpInst::FCMP_UGE:
7948 return new FCmpInst(FCmpInst::FCMP_UEQ, Op0, Op1, "", &I);
7949 }
7950 }
7951
7952 // Ignore signbit of bitcasted int when comparing equality to FP 0.0:
7953 // fcmp oeq/une (bitcast X), 0.0 --> (and X, SignMaskC) ==/!= 0
7954 if (match(Op1, m_PosZeroFP()) &&
7957 if (Pred == FCmpInst::FCMP_OEQ)
7958 IntPred = ICmpInst::ICMP_EQ;
7959 else if (Pred == FCmpInst::FCMP_UNE)
7960 IntPred = ICmpInst::ICMP_NE;
7961
7962 if (IntPred != ICmpInst::BAD_ICMP_PREDICATE) {
7963 Type *IntTy = X->getType();
7964 const APInt &SignMask = ~APInt::getSignMask(IntTy->getScalarSizeInBits());
7965 Value *MaskX = Builder.CreateAnd(X, ConstantInt::get(IntTy, SignMask));
7966 return new ICmpInst(IntPred, MaskX, ConstantInt::getNullValue(IntTy));
7967 }
7968 }
7969
7970 // Handle fcmp with instruction LHS and constant RHS.
7971 Instruction *LHSI;
7972 Constant *RHSC;
7973 if (match(Op0, m_Instruction(LHSI)) && match(Op1, m_Constant(RHSC))) {
7974 switch (LHSI->getOpcode()) {
7975 case Instruction::PHI:
7976 if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI)))
7977 return NV;
7978 break;
7979 case Instruction::SIToFP:
7980 case Instruction::UIToFP:
7981 if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC))
7982 return NV;
7983 break;
7984 case Instruction::FDiv:
7985 if (Instruction *NV = foldFCmpReciprocalAndZero(I, LHSI, RHSC))
7986 return NV;
7987 break;
7988 case Instruction::Load:
7989 if (auto *GEP = dyn_cast<GetElementPtrInst>(LHSI->getOperand(0)))
7990 if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)))
7992 cast<LoadInst>(LHSI), GEP, GV, I))
7993 return Res;
7994 break;
7995 }
7996 }
7997
7998 if (Instruction *R = foldFabsWithFcmpZero(I, *this))
7999 return R;
8000
8001 if (match(Op0, m_FNeg(m_Value(X)))) {
8002 // fcmp pred (fneg X), C --> fcmp swap(pred) X, -C
8003 Constant *C;
8004 if (match(Op1, m_Constant(C)))
8005 if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL))
8006 return new FCmpInst(I.getSwappedPredicate(), X, NegC, "", &I);
8007 }
8008
8009 if (match(Op0, m_FPExt(m_Value(X)))) {
8010 // fcmp (fpext X), (fpext Y) -> fcmp X, Y
8011 if (match(Op1, m_FPExt(m_Value(Y))) && X->getType() == Y->getType())
8012 return new FCmpInst(Pred, X, Y, "", &I);
8013
8014 const APFloat *C;
8015 if (match(Op1, m_APFloat(C))) {
8016 const fltSemantics &FPSem =
8017 X->getType()->getScalarType()->getFltSemantics();
8018 bool Lossy;
8019 APFloat TruncC = *C;
8020 TruncC.convert(FPSem, APFloat::rmNearestTiesToEven, &Lossy);
8021
8022 if (Lossy) {
8023 // X can't possibly equal the higher-precision constant, so reduce any
8024 // equality comparison.
8025 // TODO: Other predicates can be handled via getFCmpCode().
8026 switch (Pred) {
8027 case FCmpInst::FCMP_OEQ:
8028 // X is ordered and equal to an impossible constant --> false
8029 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8030 case FCmpInst::FCMP_ONE:
8031 // X is ordered and not equal to an impossible constant --> ordered
8032 return new FCmpInst(FCmpInst::FCMP_ORD, X,
8033 ConstantFP::getZero(X->getType()));
8034 case FCmpInst::FCMP_UEQ:
8035 // X is unordered or equal to an impossible constant --> unordered
8036 return new FCmpInst(FCmpInst::FCMP_UNO, X,
8037 ConstantFP::getZero(X->getType()));
8038 case FCmpInst::FCMP_UNE:
8039 // X is unordered or not equal to an impossible constant --> true
8040 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8041 default:
8042 break;
8043 }
8044 }
8045
8046 // fcmp (fpext X), C -> fcmp X, (fptrunc C) if fptrunc is lossless
8047 // Avoid lossy conversions and denormals.
8048 // Zero is a special case that's OK to convert.
8049 APFloat Fabs = TruncC;
8050 Fabs.clearSign();
8051 if (!Lossy &&
8052 (Fabs.isZero() || !(Fabs < APFloat::getSmallestNormalized(FPSem)))) {
8053 Constant *NewC = ConstantFP::get(X->getType(), TruncC);
8054 return new FCmpInst(Pred, X, NewC, "", &I);
8055 }
8056 }
8057 }
8058
8059 // Convert a sign-bit test of an FP value into a cast and integer compare.
8060 // TODO: Simplify if the copysign constant is 0.0 or NaN.
8061 // TODO: Handle non-zero compare constants.
8062 // TODO: Handle other predicates.
8063 if (match(Op0, m_OneUse(m_Intrinsic<Intrinsic::copysign>(m_APFloat(C),
8064 m_Value(X)))) &&
8065 match(Op1, m_AnyZeroFP()) && !C->isZero() && !C->isNaN()) {
8066 Type *IntType = Builder.getIntNTy(X->getType()->getScalarSizeInBits());
8067 if (auto *VecTy = dyn_cast<VectorType>(OpType))
8068 IntType = VectorType::get(IntType, VecTy->getElementCount());
8069
8070 // copysign(non-zero constant, X) < 0.0 --> (bitcast X) < 0
8071 if (Pred == FCmpInst::FCMP_OLT) {
8072 Value *IntX = Builder.CreateBitCast(X, IntType);
8073 return new ICmpInst(ICmpInst::ICMP_SLT, IntX,
8074 ConstantInt::getNullValue(IntType));
8075 }
8076 }
8077
8078 {
8079 Value *CanonLHS = nullptr, *CanonRHS = nullptr;
8080 match(Op0, m_Intrinsic<Intrinsic::canonicalize>(m_Value(CanonLHS)));
8081 match(Op1, m_Intrinsic<Intrinsic::canonicalize>(m_Value(CanonRHS)));
8082
8083 // (canonicalize(x) == x) => (x == x)
8084 if (CanonLHS == Op1)
8085 return new FCmpInst(Pred, Op1, Op1, "", &I);
8086
8087 // (x == canonicalize(x)) => (x == x)
8088 if (CanonRHS == Op0)
8089 return new FCmpInst(Pred, Op0, Op0, "", &I);
8090
8091 // (canonicalize(x) == canonicalize(y)) => (x == y)
8092 if (CanonLHS && CanonRHS)
8093 return new FCmpInst(Pred, CanonLHS, CanonRHS, "", &I);
8094 }
8095
8096 if (I.getType()->isVectorTy())
8097 if (Instruction *Res = foldVectorCmp(I, Builder))
8098 return Res;
8099
8100 return Changed ? &I : nullptr;
8101}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
Rewrite undef for PHI
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Hexagon Common GEP
#define _
static Instruction * foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI, Constant *RHSC)
Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary.
static Instruction * foldFabsWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC)
Optimize fabs(X) compared with zero.
static Instruction * foldICmpUSubSatOrUAddSatWithConstant(ICmpInst::Predicate Pred, SaturatingInst *II, const APInt &C, InstCombiner::BuilderTy &Builder)
static bool addWithOverflow(APInt &Result, const APInt &In1, const APInt &In2, bool IsSigned=false)
Compute Result = In1+In2, returning true if the result overflowed for this type.
static Value * foldICmpWithLowBitMaskedVal(ICmpInst::Predicate Pred, Value *Op0, Value *Op1, const SimplifyQuery &Q, InstCombiner &IC)
Some comparisons can be simplified.
static Instruction * foldICmpAndXX(ICmpInst &I, const SimplifyQuery &Q, InstCombinerImpl &IC)
static Instruction * foldVectorCmp(CmpInst &Cmp, InstCombiner::BuilderTy &Builder)
static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q, unsigned Depth=0)
static Value * createLogicFromTable(const std::bitset< 4 > &Table, Value *Op0, Value *Op1, IRBuilderBase &Builder, bool HasOneUse)
static Instruction * foldICmpOfUAddOv(ICmpInst &I)
static Instruction * foldICmpShlOne(ICmpInst &Cmp, Instruction *Shl, const APInt &C)
Fold icmp (shl 1, Y), C.
static bool isChainSelectCmpBranch(const SelectInst *SI)
Return true when the instruction sequence within a block is select-cmp-br.
static Instruction * foldICmpInvariantGroup(ICmpInst &I)
static Instruction * foldReductionIdiom(ICmpInst &I, InstCombiner::BuilderTy &Builder, const DataLayout &DL)
This function folds patterns produced by lowering of reduce idioms, such as llvm.vector....
static Instruction * canonicalizeICmpBool(ICmpInst &I, InstCombiner::BuilderTy &Builder)
Integer compare with boolean values can always be turned into bitwise ops.
static Value * foldICmpOrXorSubChain(ICmpInst &Cmp, BinaryOperator *Or, InstCombiner::BuilderTy &Builder)
Fold icmp eq/ne (or (xor/sub (X1, X2), xor/sub (X3, X4))), 0.
static bool hasBranchUse(ICmpInst &I)
Given an icmp instruction, return true if any use of this comparison is a branch on sign bit comparis...
static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth)
When performing a comparison against a constant, it is possible that not all the bits in the LHS are ...
static Instruction * foldICmpXorXX(ICmpInst &I, const SimplifyQuery &Q, InstCombinerImpl &IC)
static Instruction * processUMulZExtIdiom(ICmpInst &I, Value *MulVal, const APInt *OtherVal, InstCombinerImpl &IC)
Recognize and process idiom involving test for multiplication overflow.
static Instruction * transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS, ICmpInst::Predicate Cond, const DataLayout &DL, InstCombiner &IC)
Converts (CMP GEPLHS, RHS) if this change would make RHS a constant.
static Instruction * foldFCmpFNegCommonOp(FCmpInst &I)
static bool canRewriteGEPAsOffset(Value *Start, Value *Base, const DataLayout &DL, SetVector< Value * > &Explored)
Returns true if we can rewrite Start as a GEP with pointer Base and some integer offset.
static Instruction * foldICmpWithHighBitMask(ICmpInst &Cmp, InstCombiner::BuilderTy &Builder)
static ICmpInst * canonicalizeCmpWithConstant(ICmpInst &I)
If we have an icmp le or icmp ge instruction with a constant operand, turn it into the appropriate ic...
static Instruction * foldICmpIntrinsicWithIntrinsic(ICmpInst &Cmp, InstCombiner::BuilderTy &Builder)
Fold an icmp with LLVM intrinsics.
static Instruction * foldICmpPow2Test(ICmpInst &I, InstCombiner::BuilderTy &Builder)
static bool subWithOverflow(APInt &Result, const APInt &In1, const APInt &In2, bool IsSigned=false)
Compute Result = In1-In2, returning true if the result overflowed for this type.
static Instruction * foldICmpXNegX(ICmpInst &I, InstCombiner::BuilderTy &Builder)
static Instruction * processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B, ConstantInt *CI2, ConstantInt *CI1, InstCombinerImpl &IC)
The caller has matched a pattern of the form: I = icmp ugt (add (add A, B), CI2), CI1 If this is of t...
static Value * foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ, InstCombiner::BuilderTy &Builder)
static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C)
Returns true if the exploded icmp can be expressed as a signed comparison to zero and updates the pre...
static Instruction * foldCtpopPow2Test(ICmpInst &I, IntrinsicInst *CtpopLhs, const APInt &CRhs, InstCombiner::BuilderTy &Builder, const SimplifyQuery &Q)
static void setInsertionPoint(IRBuilder<> &Builder, Value *V, bool Before=true)
static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS, bool IsSigned)
static Value * foldICmpWithTruncSignExtendedVal(ICmpInst &I, InstCombiner::BuilderTy &Builder)
Some comparisons can be simplified.
static Value * rewriteGEPAsOffset(Value *Start, Value *Base, const DataLayout &DL, SetVector< Value * > &Explored, InstCombiner &IC)
Returns a re-written value of Start as an indexed GEP using Base as a pointer.
static Instruction * foldICmpOrXX(ICmpInst &I, const SimplifyQuery &Q, InstCombinerImpl &IC)
This file provides internal interfaces used to implement the InstCombine.
This file provides the interface for the instcombine pass implementation.
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:531
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
mir Rename Register Operands
#define T1
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file implements a set that has insertion order iteration characteristics.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
Value * RHS
Value * LHS
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition: APFloat.cpp:5196
void clearSign()
Definition: APFloat.h:1159
bool isZero() const
Definition: APFloat.h:1291
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
Definition: APFloat.h:1026
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition: APFloat.h:1006
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition: APFloat.h:966
FPClassTest classify() const
Return the FPClassTest which will return true for the value.
Definition: APFloat.cpp:5183
opStatus roundToIntegral(roundingMode RM)
Definition: APFloat.h:1109
Class for arbitrary precision integers.
Definition: APInt.h:76
APInt udiv(const APInt &RHS) const
Unsigned division operation.
Definition: APInt.cpp:1579
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
Definition: APInt.h:427
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:981
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition: APInt.h:401
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1491
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1463
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:906
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:184
APInt abs() const
Get the absolute value.
Definition: APInt.h:1737
unsigned ceilLogBase2() const
Definition: APInt.h:1706
bool sgt(const APInt &RHS) const
Signed greater than comparison.
Definition: APInt.h:1179
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition: APInt.h:349
APInt usub_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1954
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition: APInt.h:1160
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition: APInt.h:358
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
Definition: APInt.h:444
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1439
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.h:1089
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:187
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
Definition: APInt.h:194
bool isNegative() const
Determine sign of this APInt.
Definition: APInt.h:307
APInt sadd_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1934
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1227
bool eq(const APInt &RHS) const
Equality comparison.
Definition: APInt.h:1057
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
Definition: APInt.cpp:1650
bool sle(const APInt &RHS) const
Signed less or equal comparison.
Definition: APInt.h:1144
APInt uadd_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1941
void negate()
Negate this APInt in place.
Definition: APInt.h:1421
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition: APInt.h:1589
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition: APInt.h:1548
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:197
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Definition: APInt.h:334
unsigned countl_one() const
Count the number of leading one bits.
Definition: APInt.h:1565
unsigned logBase2() const
Definition: APInt.h:1703
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition: APInt.h:453
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition: APInt.h:805
bool isMaxSignedValue() const
Determine if this is the largest signed value.
Definition: APInt.h:383
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition: APInt.h:1128
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition: APInt.h:851
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:418
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:284
bool slt(const APInt &RHS) const
Signed less than comparison.
Definition: APInt.h:1108
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition: APInt.h:274
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition: APInt.h:178
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
Definition: APInt.h:1215
APInt ssub_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1947
bool isOne() const
Determine if this is a value of 1.
Definition: APInt.h:367
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition: APInt.h:264
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:217
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:829
unsigned countr_one() const
Count the number of trailing one bits.
Definition: APInt.h:1606
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition: APInt.h:1199
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
an instruction to allocate memory on the stack
Definition: Instructions.h:59
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
Class to represent array types.
Definition: DerivedTypes.h:371
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:396
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:439
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:220
unsigned getNoWrapKind() const
Returns one of OBO::NoSignedWrap or OBO::NoUnsignedWrap.
Instruction::BinaryOps getBinaryOp() const
Returns the binary operation underlying the intrinsic.
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore)
Construct a binary instruction, given the opcode and the two operands.
BinaryOps getOpcode() const
Definition: InstrTypes.h:491
static BinaryOperator * CreateNot(Value *Op, const Twine &Name, BasicBlock::iterator InsertBefore)
Conditional or Unconditional Branch instruction.
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1648
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr, BasicBlock::iterator InsertBefore)
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:955
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Definition: InstrTypes.h:1323
Predicate getStrictPredicate() const
For example, SGE -> SGT, SLE -> SLT, ULE -> ULT, UGE -> UGT.
Definition: InstrTypes.h:1159
bool isEquality() const
Determine if this is an equals/not equals predicate.
Definition: InstrTypes.h:1216
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:965
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:968
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:982
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:994
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:995
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:971
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:980
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:969
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:970
@ ICMP_UGE
unsigned greater or equal
Definition: InstrTypes.h:989
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:988
@ ICMP_SGT
signed greater than
Definition: InstrTypes.h:992
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:979
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:973
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:976
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:990
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:977
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:972
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:974
@ ICMP_EQ
equal
Definition: InstrTypes.h:986
@ ICMP_NE
not equal
Definition: InstrTypes.h:987
@ ICMP_SGE
signed greater or equal
Definition: InstrTypes.h:993
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:981
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:991
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:978
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:967
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:975
bool isSigned() const
Definition: InstrTypes.h:1226
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition: InstrTypes.h:1128
static CmpInst * Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore)
Construct a compare instruction, given the opcode, the predicate and the two operands.
bool isTrueWhenEqual() const
This is just a convenience.
Definition: InstrTypes.h:1275
Predicate getNonStrictPredicate() const
For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
Definition: InstrTypes.h:1172
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition: InstrTypes.h:1090
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:1066
Predicate getFlippedStrictnessPredicate() const
For predicate of kind "is X or equal to 0" returns the predicate "is X".
Definition: InstrTypes.h:1194
Predicate getFlippedSignednessPredicate()
For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->Failed assert.
Definition: InstrTypes.h:1269
bool isIntPredicate() const
Definition: InstrTypes.h:1084
bool isUnsigned() const
Definition: InstrTypes.h:1232
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2126
static Constant * getPointerBitCastOrAddrSpaceCast(Constant *C, Type *Ty)
Create a BitCast or AddrSpaceCast for a pointer type depending on the address space.
Definition: Constants.cpp:2087
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2544
static Constant * getNot(Constant *C)
Definition: Constants.cpp:2531
static Constant * getXor(Constant *C1, Constant *C2)
Definition: Constants.cpp:2558
static Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2537
static Constant * getNeg(Constant *C, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2525
static Constant * getCompare(unsigned short pred, Constant *C1, Constant *C2, bool OnlyIfReduced=false)
Return an ICmp or FCmp comparison operator constant expression.
Definition: Constants.cpp:2328
static Constant * getZero(Type *Ty, bool Negative=false)
Definition: Constants.cpp:1037
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
Definition: Constants.h:254
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:849
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:204
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
Definition: Constants.h:122
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:856
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
Definition: Constants.h:147
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:144
static ConstantInt * getBool(LLVMContext &Context, bool V)
Definition: Constants.cpp:863
This class represents a range of values.
Definition: ConstantRange.h:47
ConstantRange add(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an addition of a value in this ran...
std::optional< ConstantRange > exactUnionWith(const ConstantRange &CR) const
Union the two ranges and return the result if it can be represented exactly, otherwise return std::nu...
ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
const APInt * getSingleElement() const
If this set contains a single element, return it, otherwise return null.
ConstantRange difference(const ConstantRange &CR) const
Subtract the specified range from this range (aka relative complement of the sets).
bool isEmptySet() const
Return true if this set contains no members.
static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
ConstantRange inverse() const
Return a new range that is the logical not of the current set.
std::optional< ConstantRange > exactIntersectWith(const ConstantRange &CR) const
Intersect the two ranges and return the result if it can be represented exactly, otherwise return std...
ConstantRange intersectWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the intersection of this range with another range.
ConstantRange sub(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a subtraction of a value in this r...
static ConstantRange makeExactNoWrapRegion(Instruction::BinaryOps BinOp, const APInt &Other, unsigned NoWrapKind)
Produce the range that contains X if and only if "X BinOp Other" does not wrap.
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
Definition: Constants.cpp:1449
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
Definition: Constants.cpp:400
static Constant * replaceUndefsWith(Constant *C, Constant *Replacement)
Try to replace undefined constant C or undefined elements in C with Replacement.
Definition: Constants.cpp:767
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:417
bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
Definition: Constants.cpp:107
const APInt & getUniqueInteger() const
If C is a constant integer then return its value, otherwise C must be a vector of constant integers,...
Definition: Constants.cpp:1758
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
Definition: Constants.cpp:432
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition: Constants.cpp:90
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
bool isLegalInteger(uint64_t Width) const
Returns true if the specified type is known to be a native integer type supported by the CPU.
Definition: DataLayout.h:260
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Definition: DataLayout.cpp:878
unsigned getPointerTypeSizeInBits(Type *) const
Layout pointer size, in bits, based on the type.
Definition: DataLayout.cpp:763
IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
Definition: DataLayout.cpp:905
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:504
Type * getSmallestLegalIntType(LLVMContext &C, unsigned Width=0) const
Returns the smallest integer type with size at least as big as Width bits.
Definition: DataLayout.cpp:893
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition: DenseMap.h:145
ArrayRef< BranchInst * > conditionsFor(const Value *V) const
Access the list of branches which affect this value.
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Definition: Dominators.cpp:122
static ExtractValueInst * Create(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &NameStr, BasicBlock::iterator InsertBefore)
This instruction compares its operands according to the predicate given to the constructor.
bool isInBounds() const
Test whether this is an inbounds GEP, as defined by LangRef.html.
Definition: Operator.h:420
Type * getSourceElementType() const
Definition: Operator.cpp:60
Value * getPointerOperand()
Definition: Operator.h:445
bool hasAllConstantIndices() const
Return true if all of the indices of this GEP are constant integers.
Definition: Operator.h:492
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:973
Type * getValueType() const
Definition: GlobalValue.h:296
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool isConstant() const
If the value is a global constant, its value is immutable throughout the runtime execution of the pro...
bool hasDefinitiveInitializer() const
hasDefinitiveInitializer - Whether the global variable has an initializer, and any other instances of...
This instruction compares its operands according to the predicate given to the constructor.
static bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
Predicate getSignedPredicate() const
For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
bool isEquality() const
Return true if this predicate is either EQ or NE.
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
bool isRelational() const
Return true if the predicate is relational (not EQ or NE).
Predicate getUnsignedPredicate() const
For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
Definition: IRBuilder.cpp:913
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2240
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2006
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1715
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2443
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Definition: IRBuilder.h:533
Value * CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2248
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
Definition: IRBuilder.h:2028
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1214
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition: IRBuilder.h:2499
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition: IRBuilder.h:460
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Definition: IRBuilder.cpp:932
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1431
Value * CreateNSWAdd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1330
Value * CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
Definition: IRBuilder.h:1875
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2228
Value * createIsFPClass(Value *FPNum, unsigned Test)
Definition: IRBuilder.cpp:1309
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:480
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2349
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2380
Value * CreateNot(Value *V, const Twine &Name="")
Definition: IRBuilder.h:1748
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2224
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1338
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2110
Value * CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2232
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1410
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Definition: IRBuilder.h:2010
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1469
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1321
ConstantInt * getFalse()
Get the constant value for i1 false.
Definition: IRBuilder.h:465
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1491
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1660
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2256
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2144
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:2179
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
Definition: IRBuilder.h:2527
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition: IRBuilder.h:180
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2395
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1513
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Definition: IRBuilder.h:1865
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2334
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Definition: IRBuilder.h:510
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
Definition: IRBuilder.h:496
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1398
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1355
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2649
Instruction * foldICmpShrConstant(ICmpInst &Cmp, BinaryOperator *Shr, const APInt &C)
Fold icmp ({al}shr X, Y), C.
Instruction * foldICmpWithZextOrSext(ICmpInst &ICmp)
Instruction * foldICmpSelectConstant(ICmpInst &Cmp, SelectInst *Select, ConstantInt *C)
Instruction * foldICmpSRemConstant(ICmpInst &Cmp, BinaryOperator *UDiv, const APInt &C)
Instruction * foldICmpBinOpWithConstant(ICmpInst &Cmp, BinaryOperator *BO, const APInt &C)
Fold an icmp with BinaryOp and constant operand: icmp Pred BO, C.
Instruction * foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or, const APInt &C)
Fold icmp (or X, Y), C.
Instruction * foldICmpTruncWithTruncOrExt(ICmpInst &Cmp, const SimplifyQuery &Q)
Fold icmp (trunc X), (trunc Y).
Instruction * foldSignBitTest(ICmpInst &I)
Fold equality-comparison between zero and any (maybe truncated) right-shift by one-less-than-bitwidth...
bool SimplifyDemandedBits(Instruction *I, unsigned Op, const APInt &DemandedMask, KnownBits &Known, unsigned Depth=0) override
This form of SimplifyDemandedBits simplifies the specified instruction operand if possible,...
Instruction * foldOpIntoPhi(Instruction &I, PHINode *PN)
Given a binary operator, cast instruction, or select which has a PHI node as operand #0,...
Value * insertRangeTest(Value *V, const APInt &Lo, const APInt &Hi, bool isSigned, bool Inside)
Emit a computation of: (V >= Lo && V < Hi) if Inside is true, otherwise (V < Lo || V >= Hi).
Instruction * foldICmpBinOp(ICmpInst &Cmp, const SimplifyQuery &SQ)
Try to fold icmp (binop), X or icmp X, (binop).
Instruction * foldICmpSubConstant(ICmpInst &Cmp, BinaryOperator *Sub, const APInt &C)
Fold icmp (sub X, Y), C.
Instruction * foldICmpInstWithConstantNotInt(ICmpInst &Cmp)
Handle icmp with constant (but not simple integer constant) RHS.
Instruction * foldICmpWithMinMax(Instruction &I, MinMaxIntrinsic *MinMax, Value *Z, ICmpInst::Predicate Pred)
Fold icmp Pred min|max(X, Y), Z.
Instruction * foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, ICmpInst::Predicate Cond, Instruction &I)
Fold comparisons between a GEP instruction and something else.
Instruction * foldICmpShlConstConst(ICmpInst &I, Value *ShAmt, const APInt &C1, const APInt &C2)
Handle "(icmp eq/ne (shl AP2, A), AP1)" -> (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)).
Value * reassociateShiftAmtsOfTwoSameDirectionShifts(BinaryOperator *Sh0, const SimplifyQuery &SQ, bool AnalyzeForSignBitExtraction=false)
Instruction * foldICmpEqIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II, const APInt &C)
Fold an equality icmp with LLVM intrinsic and constant operand.
Value * foldMultiplicationOverflowCheck(ICmpInst &Cmp)
Fold (-1 u/ x) u< y ((x * y) ?/ x) != y to @llvm.
Instruction * foldICmpInstWithConstantAllowUndef(ICmpInst &Cmp, const APInt &C)
Try to fold integer comparisons with a constant operand: icmp Pred X, C where X is some kind of instr...
Instruction * foldICmpWithConstant(ICmpInst &Cmp)
Fold icmp Pred X, C.
CmpInst * canonicalizeICmpPredicate(CmpInst &I)
If we have a comparison with a non-canonical predicate, if we can update all the users,...
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Instruction * foldICmpWithZero(ICmpInst &Cmp)
Instruction * foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp, BinaryOperator *BO, const APInt &C)
Fold an icmp equality instruction with binary operator LHS and constant RHS: icmp eq/ne BO,...
Instruction * foldICmpUsingBoolRange(ICmpInst &I)
If one operand of an icmp is effectively a bool (value range of {0,1}), then try to reduce patterns b...
Instruction * foldICmpWithTrunc(ICmpInst &Cmp)
Instruction * foldICmpIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II, const APInt &C)
Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
bool matchThreeWayIntCompare(SelectInst *SI, Value *&LHS, Value *&RHS, ConstantInt *&Less, ConstantInt *&Equal, ConstantInt *&Greater)
Match a select chain which produces one of three values based on whether the LHS is less than,...
Instruction * foldCmpLoadFromIndexedGlobal(LoadInst *LI, GetElementPtrInst *GEP, GlobalVariable *GV, CmpInst &ICI, ConstantInt *AndCst=nullptr)
This is called when we see this pattern: cmp pred (load (gep GV, ...)), cmpcst where GV is a global v...
Instruction * visitFCmpInst(FCmpInst &I)
Instruction * foldICmpUsingKnownBits(ICmpInst &Cmp)
Try to fold the comparison based on range information we can get by checking whether bits are known t...
Instruction * foldICmpDivConstant(ICmpInst &Cmp, BinaryOperator *Div, const APInt &C)
Fold icmp ({su}div X, Y), C.
Instruction * foldIRemByPowerOfTwoToBitTest(ICmpInst &I)
If we have: icmp eq/ne (urem/srem x, y), 0 iff y is a power-of-two, we can replace this with a bit te...
Instruction * foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, Constant *RHSC)
Fold fcmp ([us]itofp x, cst) if possible.
Instruction * foldICmpUDivConstant(ICmpInst &Cmp, BinaryOperator *UDiv, const APInt &C)
Fold icmp (udiv X, Y), C.
Constant * getLosslessTrunc(Constant *C, Type *TruncTy, unsigned ExtOp)
Instruction * foldICmpWithCastOp(ICmpInst &ICmp)
Handle icmp (cast x), (cast or constant).
Instruction * foldICmpTruncConstant(ICmpInst &Cmp, TruncInst *Trunc, const APInt &C)
Fold icmp (trunc X), C.
Instruction * foldICmpAddConstant(ICmpInst &Cmp, BinaryOperator *Add, const APInt &C)
Fold icmp (add X, Y), C.
Instruction * foldICmpMulConstant(ICmpInst &Cmp, BinaryOperator *Mul, const APInt &C)
Fold icmp (mul X, Y), C.
Instruction * tryFoldInstWithCtpopWithNot(Instruction *I)
Instruction * foldICmpXorConstant(ICmpInst &Cmp, BinaryOperator *Xor, const APInt &C)
Fold icmp (xor X, Y), C.
Instruction * foldICmpAddOpConst(Value *X, const APInt &C, ICmpInst::Predicate Pred)
Fold "icmp pred (X+C), X".
Instruction * foldICmpAndShift(ICmpInst &Cmp, BinaryOperator *And, const APInt &C1, const APInt &C2)
Fold icmp (and (sh X, Y), C2), C1.
Instruction * foldICmpInstWithConstant(ICmpInst &Cmp)
Try to fold integer comparisons with a constant operand: icmp Pred X, C where X is some kind of instr...
Instruction * foldICmpXorShiftConst(ICmpInst &Cmp, BinaryOperator *Xor, const APInt &C)
For power-of-2 C: ((X s>> ShiftC) ^ X) u< C --> (X + C) u< (C << 1) ((X s>> ShiftC) ^ X) u> (C - 1) -...
Instruction * foldICmpShlConstant(ICmpInst &Cmp, BinaryOperator *Shl, const APInt &C)
Fold icmp (shl X, Y), C.
Instruction * foldICmpAndConstant(ICmpInst &Cmp, BinaryOperator *And, const APInt &C)
Fold icmp (and X, Y), C.
Instruction * foldICmpEquality(ICmpInst &Cmp)
bool dominatesAllUses(const Instruction *DI, const Instruction *UI, const BasicBlock *DB) const
True when DB dominates all uses of DI except UI.
bool foldAllocaCmp(AllocaInst *Alloca)
Instruction * foldICmpCommutative(ICmpInst::Predicate Pred, Value *Op0, Value *Op1, ICmpInst &CxtI)
Instruction * visitICmpInst(ICmpInst &I)
Instruction * foldSelectICmp(ICmpInst::Predicate Pred, SelectInst *SI, Value *RHS, const ICmpInst &I)
OverflowResult computeOverflow(Instruction::BinaryOps BinaryOp, bool IsSigned, Value *LHS, Value *RHS, Instruction *CxtI) const
Instruction * foldICmpWithDominatingICmp(ICmpInst &Cmp)
Canonicalize icmp instructions based on dominating conditions.
bool replacedSelectWithOperand(SelectInst *SI, const ICmpInst *Icmp, const unsigned SIOpd)
Try to replace select with select operand SIOpd in SI-ICmp sequence.
Instruction * foldICmpShrConstConst(ICmpInst &I, Value *ShAmt, const APInt &C1, const APInt &C2)
Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" -> (icmp eq/ne A, Log2(AP2/AP1)) -> (icmp eq/ne A,...
void freelyInvertAllUsersOf(Value *V, Value *IgnoredUser=nullptr)
Freely adapt every user of V as-if V was changed to !V.
Instruction * foldICmpAndConstConst(ICmpInst &Cmp, BinaryOperator *And, const APInt &C1)
Fold icmp (and X, C2), C1.
Instruction * foldICmpBitCast(ICmpInst &Cmp)
The core instruction combiner logic.
Definition: InstCombiner.h:47
OverflowResult computeOverflowForSignedSub(const Value *LHS, const Value *RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:497
SimplifyQuery SQ
Definition: InstCombiner.h:76
static bool isCanonicalPredicate(CmpInst::Predicate Pred)
Predicate canonicalization reduces the number of patterns that need to be matched by other transforms...
Definition: InstCombiner.h:156
bool isFreeToInvert(Value *V, bool WillInvertAllUses, bool &DoesConsume)
Return true if the specified value is free to invert (apply ~ to).
Definition: InstCombiner.h:231
static unsigned getComplexity(Value *V)
Assign a complexity or rank value to LLVM Values.
Definition: InstCombiner.h:138
TargetLibraryInfo & TLI
Definition: InstCombiner.h:73
bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero=false, unsigned Depth=0, const Instruction *CxtI=nullptr)
Definition: InstCombiner.h:440
OverflowResult computeOverflowForUnsignedMul(const Value *LHS, const Value *RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:461
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
Definition: InstCombiner.h:385
uint64_t MaxArraySizeForCombine
Maximum size of array considered when transforming.
Definition: InstCombiner.h:55
OverflowResult computeOverflowForSignedAdd(const WithCache< const Value * > &LHS, const WithCache< const Value * > &RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:483
static Constant * SubOne(Constant *C)
Subtract one from a Constant.
Definition: InstCombiner.h:179
static std::optional< std::pair< CmpInst::Predicate, Constant * > > getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred, Constant *C)
OverflowResult computeOverflowForUnsignedSub(const Value *LHS, const Value *RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:490
const DataLayout & DL
Definition: InstCombiner.h:75
DomConditionCache DC
Definition: InstCombiner.h:80
bool canFreelyInvertAllUsersOf(Instruction *V, Value *IgnoredUser)
Given i1 V, can every user of V be freely adapted if V is changed to !V ? InstCombine's freelyInvertA...
Definition: InstCombiner.h:247
void addToWorklist(Instruction *I)
Definition: InstCombiner.h:335
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
Definition: InstCombiner.h:409
DominatorTree & DT
Definition: InstCombiner.h:74
OverflowResult computeOverflowForSignedMul(const Value *LHS, const Value *RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:468
void computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth, const Instruction *CxtI) const
Definition: InstCombiner.h:430
BuilderTy & Builder
Definition: InstCombiner.h:60
OverflowResult computeOverflowForUnsignedAdd(const WithCache< const Value * > &LHS, const WithCache< const Value * > &RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:475
Value * getFreelyInverted(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume)
Definition: InstCombiner.h:212
const SimplifyQuery & getSimplifyQuery() const
Definition: InstCombiner.h:341
unsigned ComputeMaxSignificantBits(const Value *Op, unsigned Depth=0, const Instruction *CxtI=nullptr) const
Definition: InstCombiner.h:456
bool hasNoUnsignedWrap() const LLVM_READONLY
Determine whether the no unsigned wrap flag is set.
bool hasNoInfs() const LLVM_READONLY
Determine whether the no-infs flag is set.
bool isArithmeticShift() const
Return true if this is an arithmetic shift right.
Definition: Instruction.h:295
bool hasNoSignedWrap() const LLVM_READONLY
Determine whether the no signed wrap flag is set.
bool isCommutative() const LLVM_READONLY
Return true if the instruction is commutative:
const BasicBlock * getParent() const
Definition: Instruction.h:151
bool isExact() const LLVM_READONLY
Determine whether the exact flag is set.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:251
bool isShift() const
Definition: Instruction.h:258
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:278
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:47
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
An instruction for reading from memory.
Definition: Instructions.h:184
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: MapVector.h:141
This class represents min/max intrinsics.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr, BasicBlock::iterator InsertBefore)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Represents a saturating add/sub intrinsic.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr, BasicBlock::iterator InsertBefore, Instruction *MDFrom=nullptr)
A vector that has set insertion semantics.
Definition: SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
Definition: SetVector.h:254
This instruction constructs a fixed permutation of two input vectors.
bool empty() const
Definition: SmallVector.h:94
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
Class to represent struct types.
Definition: DerivedTypes.h:216
This class represents a truncation of integer types.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
unsigned getIntegerBitWidth() const
const fltSemantics & getFltSemantics() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition: Type.h:234
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:255
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
Definition: Type.h:166
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
int getFPMantissaWidth() const
Return the width of the mantissa of this type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:228
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
bool isIEEELikeFPTy() const
Return true if this is a well-behaved IEEE-like type, which has a IEEE compatible layout as defined b...
Definition: Type.h:171
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:348
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
Value * getOperand(unsigned i) const
Definition: User.h:169
unsigned getNumOperands() const
Definition: User.h:191
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr) const
Accumulate the constant offset this value has compared to a base pointer.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
iterator_range< user_iterator > users()
Definition: Value.h:421
bool hasNUsesOrMore(unsigned N) const
Return true if this value has N uses or more.
Definition: Value.cpp:153
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:693
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
iterator_range< use_iterator > uses()
Definition: Value.h:376
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:383
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:676
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:187
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt RoundingUDiv(const APInt &A, const APInt &B, APInt::Rounding RM)
Return A unsign-divided by B, rounded by the given rounding mode.
Definition: APInt.cpp:2765
APInt RoundingSDiv(const APInt &A, const APInt &B, APInt::Rounding RM)
Return A sign-divided by B, rounded by the given rounding mode.
Definition: APInt.cpp:2783
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1451
cst_pred_ty< is_all_ones > m_AllOnes()
Match an integer or vector with all bits set.
Definition: PatternMatch.h:477
cst_pred_ty< is_lowbit_mask > m_LowBitMask()
Match an integer or vector with only the low bit(s) set.
Definition: PatternMatch.h:622
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
specific_intval< false > m_SpecificInt(APInt V)
Match a specific integer value or vector with all elements equal to the value.
Definition: PatternMatch.h:903
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
Definition: PatternMatch.h:100
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
apint_match m_APIntAllowUndef(const APInt *&Res)
Match APInt while allowing undefs in splat vector constants.
Definition: PatternMatch.h:300
cst_pred_ty< is_power2 > m_Power2()
Match an integer or vector power-of-2.
Definition: PatternMatch.h:568
BinaryOp_match< LHS, RHS, Instruction::URem > m_URem(const LHS &L, const RHS &R)
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
Definition: PatternMatch.h:160
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, ZExtInst >, OpTy > m_ZExtOrSelf(const OpTy &Op)
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
BinOpPred_match< LHS, RHS, is_idiv_op > m_IDiv(const LHS &L, const RHS &R)
Matches integer division operations.
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
Definition: PatternMatch.h:765
cstfp_pred_ty< is_any_zero_fp > m_AnyZeroFP()
Match a floating-point negative zero or positive zero.
Definition: PatternMatch.h:713
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:821
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
OverflowingBinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub, OverflowingBinaryOperator::NoSignedWrap > m_NSWNeg(const ValTy &V)
Matches a 'Neg' as 'sub nsw 0, V'.
cst_pred_ty< is_nonnegative > m_NonNegative()
Match an integer or vector of non-negative values.
Definition: PatternMatch.h:509
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
Definition: PatternMatch.h:163
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
Definition: PatternMatch.h:541
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
BinOpPred_match< LHS, RHS, is_logical_shift_op > m_LogicalShift(const LHS &L, const RHS &R)
Matches logical shift operations.
match_combine_and< LTy, RTy > m_CombineAnd(const LTy &L, const RTy &R)
Combine two pattern matchers matching L && R.
Definition: PatternMatch.h:240
MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > m_SMin(const LHS &L, const RHS &R)
CastOperator_match< OpTy, Instruction::Trunc > m_Trunc(const OpTy &Op)
Matches Trunc.
BinaryOp_match< LHS, RHS, Instruction::Xor, true > m_c_Xor(const LHS &L, const RHS &R)
Matches an Xor with LHS and RHS in either order.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
Definition: PatternMatch.h:839
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
Definition: PatternMatch.h:548
CmpClass_match< LHS, RHS, ICmpInst, ICmpInst::Predicate > m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R)
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
match_combine_and< class_match< Constant >, match_unless< constantexpr_match > > m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
Definition: PatternMatch.h:800
CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Mul, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWMul(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::UDiv > m_UDiv(const LHS &L, const RHS &R)
cst_pred_ty< is_negated_power2_or_zero > m_NegatedPower2OrZero()
Match a integer or vector negated power-of-2.
Definition: PatternMatch.h:588
CmpClass_match< LHS, RHS, ICmpInst, ICmpInst::Predicate, true > m_c_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R)
Matches an ICmp with a predicate over LHS and RHS in either order.
cst_pred_ty< is_lowbit_mask_or_zero > m_LowBitMaskOrZero()
Match an integer or vector with only the low bit(s) set.
Definition: PatternMatch.h:632
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
match_combine_or< BinaryOp_match< LHS, RHS, Instruction::Add >, DisjointOr_match< LHS, RHS > > m_AddLike(const LHS &L, const RHS &R)
Match either "add" or "or disjoint".
CastInst_match< OpTy, UIToFPInst > m_UIToFP(const OpTy &Op)
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
match_combine_or< CastOperator_match< OpTy, Instruction::Trunc >, OpTy > m_TruncOrSelf(const OpTy &Op)
specific_intval< true > m_SpecificIntAllowUndef(APInt V)
Definition: PatternMatch.h:911
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
Definition: PatternMatch.h:294
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
Signum_match< Val_t > m_Signum(const Val_t &V)
Matches a signum pattern.
CastInst_match< OpTy, SIToFPInst > m_SIToFP(const OpTy &Op)
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
cstfp_pred_ty< is_pos_zero_fp > m_PosZeroFP()
Match a floating-point positive zero.
Definition: PatternMatch.h:722
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
UAddWithOverflow_match< LHS_t, RHS_t, Sum_t > m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S)
Match an icmp instruction checking for unsigned overflow on addition.
m_Intrinsic_Ty< Opnd0 >::Ty m_VecReverse(const Opnd0 &Op0)
BinOpPred_match< LHS, RHS, is_irem_op > m_IRem(const LHS &L, const RHS &R)
Matches integer remainder operations.
apfloat_match m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
Definition: PatternMatch.h:311
match_combine_or< match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > >, match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > > > m_MaxOrMin(const LHS &L, const RHS &R)
CastInst_match< OpTy, FPTruncInst > m_FPTrunc(const OpTy &Op)
auto m_Undef()
Match an arbitrary undef constant.
Definition: PatternMatch.h:152
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
Definition: PatternMatch.h:561
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
ElementWiseBitCast_match< OpTy > m_ElementWiseBitCast(const OpTy &Op)
m_Intrinsic_Ty< Opnd0 >::Ty m_FAbs(const Opnd0 &Op0)
BinaryOp_match< LHS, RHS, Instruction::Mul, true > m_c_Mul(const LHS &L, const RHS &R)
Matches a Mul with LHS and RHS in either order.
CastOperator_match< OpTy, Instruction::PtrToInt > m_PtrToInt(const OpTy &Op)
Matches PtrToInt.
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
match_unless< Ty > m_Unless(const Ty &M)
Match if the inner matcher does NOT match.
Definition: PatternMatch.h:198
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
Definition: PatternMatch.h:234
cst_pred_ty< icmp_pred_with_threshold > m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold)
Match an integer or vector with every element comparing 'pred' (eg/ne/...) to Threshold.
Definition: PatternMatch.h:647
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition: STLExtras.h:862
OverflowResult
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
bool isKnownNonZero(const Value *V, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given value is known to be non-zero when defined.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1731
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
bool isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS, bool &TrueIfSigned)
Given an exploded icmp instruction, return true if the comparison only checks the sign bit.
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments and pointer casts from the specified value,...
Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:665
bool isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL, bool OrZero=false, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given value is known to have exactly one bit set when defined.
Constant * ConstantFoldExtractValueInstruction(Constant *Agg, ArrayRef< unsigned > Idxs)
Attempt to constant fold an extractvalue instruction with the specified operands and indices.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
Value * simplifyAddInst(Value *LHS, Value *RHS, bool IsNSW, bool IsNUW, const SimplifyQuery &Q)
Given operands for an Add, fold the result or return null.
bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:313
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition: bit.h:281
Value * emitGEPOffset(IRBuilderBase *Builder, const DataLayout &DL, User *GEP, bool NoAssumptions=false)
Given a getelementptr instruction/constantexpr, emit the code necessary to compute the offset from th...
Definition: Local.cpp:22
constexpr unsigned MaxAnalysisRecursionDepth
Definition: ValueTracking.h:47
Constant * ConstantFoldUnaryOpOperand(unsigned Opcode, Constant *Op, const DataLayout &DL)
Attempt to constant fold a unary operation with the specified operand.
SelectPatternFlavor
Specific patterns of select instructions we can match.
@ SPF_UNKNOWN
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
bool PointerMayBeCaptured(const Value *V, bool ReturnCaptures, bool StoreCaptures, unsigned MaxUsesToExplore=0)
PointerMayBeCaptured - Return true if this pointer value may be captured by the enclosing function (w...
SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
Definition: Function.cpp:2014
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1745
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
Constant * ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL)
Attempt to constant fold a binary operation with the specified operands.
Value * simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for an ICmpInst, fold the result or return null.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
@ Add
Sum of integers.
@ UMax
Unsigned integer max implemented in terms of select(cmp()).
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
DWARFExpression::Operation Op
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:191
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition: STLExtras.h:1930
bool decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate &Pred, Value *&X, APInt &Mask, bool LookThroughTrunc=true)
Decompose an icmp into the form ((X & Mask) pred 0) if possible.
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition: STLExtras.h:2048
bool isKnownNeverNaN(const Value *V, unsigned Depth, const SimplifyQuery &SQ)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
bool isKnownPositive(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the given value is known be positive (i.e.
Value * simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q)
Given operands for an FCmpInst, fold the result or return null.
bool isKnownNonNegative(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the give value is known to be non-negative.
std::optional< bool > isImpliedCondition(const Value *LHS, const Value *RHS, const DataLayout &DL, bool LHSIsTrue=true, unsigned Depth=0)
Return true if RHS is known to be implied true by LHS.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define NC
Definition: regutils.h:42
static constexpr roundingMode rmNearestTiesToEven
Definition: APFloat.h:230
static constexpr roundingMode rmTowardZero
Definition: APFloat.h:234
This callback is used in conjunction with PointerMayBeCaptured.
Represent subnormal handling kind for floating point instruction inputs and outputs.
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ PositiveZero
Denormals are flushed to positive zero.
bool isZero() const
Returns true if value is all zero.
Definition: KnownBits.h:77
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition: KnownBits.h:238
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
Definition: KnownBits.h:270
APInt getSignedMaxValue() const
Return the maximal signed value possible given these KnownBits.
Definition: KnownBits.h:147
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition: KnownBits.h:285
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:40
bool isConstant() const
Returns true if we know the value of all bits.
Definition: KnownBits.h:50
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
Definition: KnownBits.h:292
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:244
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition: KnownBits.h:141
APInt getMinValue() const
Return the minimal unsigned value possible given these KnownBits.
Definition: KnownBits.h:125
unsigned countMinPopulation() const
Returns the number of bits known to be one.
Definition: KnownBits.h:282
APInt getSignedMinValue() const
Return the minimal signed value possible given these KnownBits.
Definition: KnownBits.h:131
const APInt & getConstant() const
Returns the value when all bits have a known value.
Definition: KnownBits.h:57
SelectPatternFlavor Flavor
static bool isMinOrMax(SelectPatternFlavor SPF)
When implementing this min/max pattern as fcmp; select, does the fcmp have to be ordered?
const DataLayout & DL
Definition: SimplifyQuery.h:61
const Instruction * CxtI
Definition: SimplifyQuery.h:65
const DominatorTree * DT
Definition: SimplifyQuery.h:63
SimplifyQuery getWithInstruction(const Instruction *I) const
Definition: SimplifyQuery.h:96
AssumptionCache * AC
Definition: SimplifyQuery.h:64
const DomConditionCache * DC
Definition: SimplifyQuery.h:66
A MapVector that performs no allocations if smaller than a certain size.
Definition: MapVector.h:254