LLVM 20.0.0git
InstCombineCompares.cpp
Go to the documentation of this file.
1//===- InstCombineCompares.cpp --------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitICmp and visitFCmp functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/ADT/APSInt.h"
15#include "llvm/ADT/SetVector.h"
16#include "llvm/ADT/Statistic.h"
24#include "llvm/IR/DataLayout.h"
25#include "llvm/IR/InstrTypes.h"
30#include <bitset>
31
32using namespace llvm;
33using namespace PatternMatch;
34
35#define DEBUG_TYPE "instcombine"
36
37// How many times is a select replaced by one of its operands?
38STATISTIC(NumSel, "Number of select opts");
39
40
41/// Compute Result = In1+In2, returning true if the result overflowed for this
42/// type.
43static bool addWithOverflow(APInt &Result, const APInt &In1,
44 const APInt &In2, bool IsSigned = false) {
45 bool Overflow;
46 if (IsSigned)
47 Result = In1.sadd_ov(In2, Overflow);
48 else
49 Result = In1.uadd_ov(In2, Overflow);
50
51 return Overflow;
52}
53
54/// Compute Result = In1-In2, returning true if the result overflowed for this
55/// type.
56static bool subWithOverflow(APInt &Result, const APInt &In1,
57 const APInt &In2, bool IsSigned = false) {
58 bool Overflow;
59 if (IsSigned)
60 Result = In1.ssub_ov(In2, Overflow);
61 else
62 Result = In1.usub_ov(In2, Overflow);
63
64 return Overflow;
65}
66
67/// Given an icmp instruction, return true if any use of this comparison is a
68/// branch on sign bit comparison.
69static bool hasBranchUse(ICmpInst &I) {
70 for (auto *U : I.users())
71 if (isa<BranchInst>(U))
72 return true;
73 return false;
74}
75
76/// Returns true if the exploded icmp can be expressed as a signed comparison
77/// to zero and updates the predicate accordingly.
78/// The signedness of the comparison is preserved.
79/// TODO: Refactor with decomposeBitTestICmp()?
80static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) {
81 if (!ICmpInst::isSigned(Pred))
82 return false;
83
84 if (C.isZero())
85 return ICmpInst::isRelational(Pred);
86
87 if (C.isOne()) {
88 if (Pred == ICmpInst::ICMP_SLT) {
89 Pred = ICmpInst::ICMP_SLE;
90 return true;
91 }
92 } else if (C.isAllOnes()) {
93 if (Pred == ICmpInst::ICMP_SGT) {
94 Pred = ICmpInst::ICMP_SGE;
95 return true;
96 }
97 }
98
99 return false;
100}
101
102/// This is called when we see this pattern:
103/// cmp pred (load (gep GV, ...)), cmpcst
104/// where GV is a global variable with a constant initializer. Try to simplify
105/// this into some simple computation that does not need the load. For example
106/// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3".
107///
108/// If AndCst is non-null, then the loaded value is masked with that constant
109/// before doing the comparison. This handles cases like "A[i]&4 == 0".
112 ConstantInt *AndCst) {
113 if (LI->isVolatile() || LI->getType() != GEP->getResultElementType() ||
114 GV->getValueType() != GEP->getSourceElementType() || !GV->isConstant() ||
116 return nullptr;
117
119 if (!isa<ConstantArray>(Init) && !isa<ConstantDataArray>(Init))
120 return nullptr;
121
122 uint64_t ArrayElementCount = Init->getType()->getArrayNumElements();
123 // Don't blow up on huge arrays.
124 if (ArrayElementCount > MaxArraySizeForCombine)
125 return nullptr;
126
127 // There are many forms of this optimization we can handle, for now, just do
128 // the simple index into a single-dimensional array.
129 //
130 // Require: GEP GV, 0, i {{, constant indices}}
131 if (GEP->getNumOperands() < 3 || !isa<ConstantInt>(GEP->getOperand(1)) ||
132 !cast<ConstantInt>(GEP->getOperand(1))->isZero() ||
133 isa<Constant>(GEP->getOperand(2)))
134 return nullptr;
135
136 // Check that indices after the variable are constants and in-range for the
137 // type they index. Collect the indices. This is typically for arrays of
138 // structs.
139 SmallVector<unsigned, 4> LaterIndices;
140
141 Type *EltTy = Init->getType()->getArrayElementType();
142 for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) {
143 ConstantInt *Idx = dyn_cast<ConstantInt>(GEP->getOperand(i));
144 if (!Idx)
145 return nullptr; // Variable index.
146
147 uint64_t IdxVal = Idx->getZExtValue();
148 if ((unsigned)IdxVal != IdxVal)
149 return nullptr; // Too large array index.
150
151 if (StructType *STy = dyn_cast<StructType>(EltTy))
152 EltTy = STy->getElementType(IdxVal);
153 else if (ArrayType *ATy = dyn_cast<ArrayType>(EltTy)) {
154 if (IdxVal >= ATy->getNumElements())
155 return nullptr;
156 EltTy = ATy->getElementType();
157 } else {
158 return nullptr; // Unknown type.
159 }
160
161 LaterIndices.push_back(IdxVal);
162 }
163
164 enum { Overdefined = -3, Undefined = -2 };
165
166 // Variables for our state machines.
167
168 // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form
169 // "i == 47 | i == 87", where 47 is the first index the condition is true for,
170 // and 87 is the second (and last) index. FirstTrueElement is -2 when
171 // undefined, otherwise set to the first true element. SecondTrueElement is
172 // -2 when undefined, -3 when overdefined and >= 0 when that index is true.
173 int FirstTrueElement = Undefined, SecondTrueElement = Undefined;
174
175 // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the
176 // form "i != 47 & i != 87". Same state transitions as for true elements.
177 int FirstFalseElement = Undefined, SecondFalseElement = Undefined;
178
179 /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these
180 /// define a state machine that triggers for ranges of values that the index
181 /// is true or false for. This triggers on things like "abbbbc"[i] == 'b'.
182 /// This is -2 when undefined, -3 when overdefined, and otherwise the last
183 /// index in the range (inclusive). We use -2 for undefined here because we
184 /// use relative comparisons and don't want 0-1 to match -1.
185 int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined;
186
187 // MagicBitvector - This is a magic bitvector where we set a bit if the
188 // comparison is true for element 'i'. If there are 64 elements or less in
189 // the array, this will fully represent all the comparison results.
190 uint64_t MagicBitvector = 0;
191
192 // Scan the array and see if one of our patterns matches.
193 Constant *CompareRHS = cast<Constant>(ICI.getOperand(1));
194 for (unsigned i = 0, e = ArrayElementCount; i != e; ++i) {
195 Constant *Elt = Init->getAggregateElement(i);
196 if (!Elt)
197 return nullptr;
198
199 // If this is indexing an array of structures, get the structure element.
200 if (!LaterIndices.empty()) {
201 Elt = ConstantFoldExtractValueInstruction(Elt, LaterIndices);
202 if (!Elt)
203 return nullptr;
204 }
205
206 // If the element is masked, handle it.
207 if (AndCst) {
208 Elt = ConstantFoldBinaryOpOperands(Instruction::And, Elt, AndCst, DL);
209 if (!Elt)
210 return nullptr;
211 }
212
213 // Find out if the comparison would be true or false for the i'th element.
215 CompareRHS, DL, &TLI);
216 if (!C)
217 return nullptr;
218
219 // If the result is undef for this element, ignore it.
220 if (isa<UndefValue>(C)) {
221 // Extend range state machines to cover this element in case there is an
222 // undef in the middle of the range.
223 if (TrueRangeEnd == (int)i - 1)
224 TrueRangeEnd = i;
225 if (FalseRangeEnd == (int)i - 1)
226 FalseRangeEnd = i;
227 continue;
228 }
229
230 // If we can't compute the result for any of the elements, we have to give
231 // up evaluating the entire conditional.
232 if (!isa<ConstantInt>(C))
233 return nullptr;
234
235 // Otherwise, we know if the comparison is true or false for this element,
236 // update our state machines.
237 bool IsTrueForElt = !cast<ConstantInt>(C)->isZero();
238
239 // State machine for single/double/range index comparison.
240 if (IsTrueForElt) {
241 // Update the TrueElement state machine.
242 if (FirstTrueElement == Undefined)
243 FirstTrueElement = TrueRangeEnd = i; // First true element.
244 else {
245 // Update double-compare state machine.
246 if (SecondTrueElement == Undefined)
247 SecondTrueElement = i;
248 else
249 SecondTrueElement = Overdefined;
250
251 // Update range state machine.
252 if (TrueRangeEnd == (int)i - 1)
253 TrueRangeEnd = i;
254 else
255 TrueRangeEnd = Overdefined;
256 }
257 } else {
258 // Update the FalseElement state machine.
259 if (FirstFalseElement == Undefined)
260 FirstFalseElement = FalseRangeEnd = i; // First false element.
261 else {
262 // Update double-compare state machine.
263 if (SecondFalseElement == Undefined)
264 SecondFalseElement = i;
265 else
266 SecondFalseElement = Overdefined;
267
268 // Update range state machine.
269 if (FalseRangeEnd == (int)i - 1)
270 FalseRangeEnd = i;
271 else
272 FalseRangeEnd = Overdefined;
273 }
274 }
275
276 // If this element is in range, update our magic bitvector.
277 if (i < 64 && IsTrueForElt)
278 MagicBitvector |= 1ULL << i;
279
280 // If all of our states become overdefined, bail out early. Since the
281 // predicate is expensive, only check it every 8 elements. This is only
282 // really useful for really huge arrays.
283 if ((i & 8) == 0 && i >= 64 && SecondTrueElement == Overdefined &&
284 SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined &&
285 FalseRangeEnd == Overdefined)
286 return nullptr;
287 }
288
289 // Now that we've scanned the entire array, emit our new comparison(s). We
290 // order the state machines in complexity of the generated code.
291 Value *Idx = GEP->getOperand(2);
292
293 // If the index is larger than the pointer offset size of the target, truncate
294 // the index down like the GEP would do implicitly. We don't have to do this
295 // for an inbounds GEP because the index can't be out of range.
296 if (!GEP->isInBounds()) {
297 Type *PtrIdxTy = DL.getIndexType(GEP->getType());
298 unsigned OffsetSize = PtrIdxTy->getIntegerBitWidth();
299 if (Idx->getType()->getPrimitiveSizeInBits().getFixedValue() > OffsetSize)
300 Idx = Builder.CreateTrunc(Idx, PtrIdxTy);
301 }
302
303 // If inbounds keyword is not present, Idx * ElementSize can overflow.
304 // Let's assume that ElementSize is 2 and the wanted value is at offset 0.
305 // Then, there are two possible values for Idx to match offset 0:
306 // 0x00..00, 0x80..00.
307 // Emitting 'icmp eq Idx, 0' isn't correct in this case because the
308 // comparison is false if Idx was 0x80..00.
309 // We need to erase the highest countTrailingZeros(ElementSize) bits of Idx.
310 unsigned ElementSize =
311 DL.getTypeAllocSize(Init->getType()->getArrayElementType());
312 auto MaskIdx = [&](Value *Idx) {
313 if (!GEP->isInBounds() && llvm::countr_zero(ElementSize) != 0) {
314 Value *Mask = Constant::getAllOnesValue(Idx->getType());
315 Mask = Builder.CreateLShr(Mask, llvm::countr_zero(ElementSize));
316 Idx = Builder.CreateAnd(Idx, Mask);
317 }
318 return Idx;
319 };
320
321 // If the comparison is only true for one or two elements, emit direct
322 // comparisons.
323 if (SecondTrueElement != Overdefined) {
324 Idx = MaskIdx(Idx);
325 // None true -> false.
326 if (FirstTrueElement == Undefined)
327 return replaceInstUsesWith(ICI, Builder.getFalse());
328
329 Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement);
330
331 // True for one element -> 'i == 47'.
332 if (SecondTrueElement == Undefined)
333 return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx);
334
335 // True for two elements -> 'i == 47 | i == 72'.
336 Value *C1 = Builder.CreateICmpEQ(Idx, FirstTrueIdx);
337 Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement);
338 Value *C2 = Builder.CreateICmpEQ(Idx, SecondTrueIdx);
339 return BinaryOperator::CreateOr(C1, C2);
340 }
341
342 // If the comparison is only false for one or two elements, emit direct
343 // comparisons.
344 if (SecondFalseElement != Overdefined) {
345 Idx = MaskIdx(Idx);
346 // None false -> true.
347 if (FirstFalseElement == Undefined)
348 return replaceInstUsesWith(ICI, Builder.getTrue());
349
350 Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement);
351
352 // False for one element -> 'i != 47'.
353 if (SecondFalseElement == Undefined)
354 return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx);
355
356 // False for two elements -> 'i != 47 & i != 72'.
357 Value *C1 = Builder.CreateICmpNE(Idx, FirstFalseIdx);
358 Value *SecondFalseIdx =
359 ConstantInt::get(Idx->getType(), SecondFalseElement);
360 Value *C2 = Builder.CreateICmpNE(Idx, SecondFalseIdx);
361 return BinaryOperator::CreateAnd(C1, C2);
362 }
363
364 // If the comparison can be replaced with a range comparison for the elements
365 // where it is true, emit the range check.
366 if (TrueRangeEnd != Overdefined) {
367 assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare");
368 Idx = MaskIdx(Idx);
369
370 // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1).
371 if (FirstTrueElement) {
372 Value *Offs = ConstantInt::get(Idx->getType(), -FirstTrueElement);
373 Idx = Builder.CreateAdd(Idx, Offs);
374 }
375
376 Value *End =
377 ConstantInt::get(Idx->getType(), TrueRangeEnd - FirstTrueElement + 1);
378 return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End);
379 }
380
381 // False range check.
382 if (FalseRangeEnd != Overdefined) {
383 assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare");
384 Idx = MaskIdx(Idx);
385 // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse).
386 if (FirstFalseElement) {
387 Value *Offs = ConstantInt::get(Idx->getType(), -FirstFalseElement);
388 Idx = Builder.CreateAdd(Idx, Offs);
389 }
390
391 Value *End =
392 ConstantInt::get(Idx->getType(), FalseRangeEnd - FirstFalseElement);
393 return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End);
394 }
395
396 // If a magic bitvector captures the entire comparison state
397 // of this load, replace it with computation that does:
398 // ((magic_cst >> i) & 1) != 0
399 {
400 Type *Ty = nullptr;
401
402 // Look for an appropriate type:
403 // - The type of Idx if the magic fits
404 // - The smallest fitting legal type
405 if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth())
406 Ty = Idx->getType();
407 else
408 Ty = DL.getSmallestLegalIntType(Init->getContext(), ArrayElementCount);
409
410 if (Ty) {
411 Idx = MaskIdx(Idx);
412 Value *V = Builder.CreateIntCast(Idx, Ty, false);
413 V = Builder.CreateLShr(ConstantInt::get(Ty, MagicBitvector), V);
414 V = Builder.CreateAnd(ConstantInt::get(Ty, 1), V);
415 return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0));
416 }
417 }
418
419 return nullptr;
420}
421
422/// Returns true if we can rewrite Start as a GEP with pointer Base
423/// and some integer offset. The nodes that need to be re-written
424/// for this transformation will be added to Explored.
426 const DataLayout &DL,
427 SetVector<Value *> &Explored) {
428 SmallVector<Value *, 16> WorkList(1, Start);
429 Explored.insert(Base);
430
431 // The following traversal gives us an order which can be used
432 // when doing the final transformation. Since in the final
433 // transformation we create the PHI replacement instructions first,
434 // we don't have to get them in any particular order.
435 //
436 // However, for other instructions we will have to traverse the
437 // operands of an instruction first, which means that we have to
438 // do a post-order traversal.
439 while (!WorkList.empty()) {
441
442 while (!WorkList.empty()) {
443 if (Explored.size() >= 100)
444 return false;
445
446 Value *V = WorkList.back();
447
448 if (Explored.contains(V)) {
449 WorkList.pop_back();
450 continue;
451 }
452
453 if (!isa<GetElementPtrInst>(V) && !isa<PHINode>(V))
454 // We've found some value that we can't explore which is different from
455 // the base. Therefore we can't do this transformation.
456 return false;
457
458 if (auto *GEP = dyn_cast<GEPOperator>(V)) {
459 // Only allow inbounds GEPs with at most one variable offset.
460 auto IsNonConst = [](Value *V) { return !isa<ConstantInt>(V); };
461 if (!GEP->isInBounds() || count_if(GEP->indices(), IsNonConst) > 1)
462 return false;
463
464 NW = NW.intersectForOffsetAdd(GEP->getNoWrapFlags());
465 if (!Explored.contains(GEP->getOperand(0)))
466 WorkList.push_back(GEP->getOperand(0));
467 }
468
469 if (WorkList.back() == V) {
470 WorkList.pop_back();
471 // We've finished visiting this node, mark it as such.
472 Explored.insert(V);
473 }
474
475 if (auto *PN = dyn_cast<PHINode>(V)) {
476 // We cannot transform PHIs on unsplittable basic blocks.
477 if (isa<CatchSwitchInst>(PN->getParent()->getTerminator()))
478 return false;
479 Explored.insert(PN);
480 PHIs.insert(PN);
481 }
482 }
483
484 // Explore the PHI nodes further.
485 for (auto *PN : PHIs)
486 for (Value *Op : PN->incoming_values())
487 if (!Explored.contains(Op))
488 WorkList.push_back(Op);
489 }
490
491 // Make sure that we can do this. Since we can't insert GEPs in a basic
492 // block before a PHI node, we can't easily do this transformation if
493 // we have PHI node users of transformed instructions.
494 for (Value *Val : Explored) {
495 for (Value *Use : Val->uses()) {
496
497 auto *PHI = dyn_cast<PHINode>(Use);
498 auto *Inst = dyn_cast<Instruction>(Val);
499
500 if (Inst == Base || Inst == PHI || !Inst || !PHI ||
501 !Explored.contains(PHI))
502 continue;
503
504 if (PHI->getParent() == Inst->getParent())
505 return false;
506 }
507 }
508 return true;
509}
510
511// Sets the appropriate insert point on Builder where we can add
512// a replacement Instruction for V (if that is possible).
513static void setInsertionPoint(IRBuilder<> &Builder, Value *V,
514 bool Before = true) {
515 if (auto *PHI = dyn_cast<PHINode>(V)) {
516 BasicBlock *Parent = PHI->getParent();
517 Builder.SetInsertPoint(Parent, Parent->getFirstInsertionPt());
518 return;
519 }
520 if (auto *I = dyn_cast<Instruction>(V)) {
521 if (!Before)
522 I = &*std::next(I->getIterator());
523 Builder.SetInsertPoint(I);
524 return;
525 }
526 if (auto *A = dyn_cast<Argument>(V)) {
527 // Set the insertion point in the entry block.
528 BasicBlock &Entry = A->getParent()->getEntryBlock();
529 Builder.SetInsertPoint(&Entry, Entry.getFirstInsertionPt());
530 return;
531 }
532 // Otherwise, this is a constant and we don't need to set a new
533 // insertion point.
534 assert(isa<Constant>(V) && "Setting insertion point for unknown value!");
535}
536
537/// Returns a re-written value of Start as an indexed GEP using Base as a
538/// pointer.
540 const DataLayout &DL,
541 SetVector<Value *> &Explored,
542 InstCombiner &IC) {
543 // Perform all the substitutions. This is a bit tricky because we can
544 // have cycles in our use-def chains.
545 // 1. Create the PHI nodes without any incoming values.
546 // 2. Create all the other values.
547 // 3. Add the edges for the PHI nodes.
548 // 4. Emit GEPs to get the original pointers.
549 // 5. Remove the original instructions.
550 Type *IndexType = IntegerType::get(
551 Base->getContext(), DL.getIndexTypeSizeInBits(Start->getType()));
552
554 NewInsts[Base] = ConstantInt::getNullValue(IndexType);
555
556 // Create the new PHI nodes, without adding any incoming values.
557 for (Value *Val : Explored) {
558 if (Val == Base)
559 continue;
560 // Create empty phi nodes. This avoids cyclic dependencies when creating
561 // the remaining instructions.
562 if (auto *PHI = dyn_cast<PHINode>(Val))
563 NewInsts[PHI] =
564 PHINode::Create(IndexType, PHI->getNumIncomingValues(),
565 PHI->getName() + ".idx", PHI->getIterator());
566 }
567 IRBuilder<> Builder(Base->getContext());
568
569 // Create all the other instructions.
570 for (Value *Val : Explored) {
571 if (NewInsts.contains(Val))
572 continue;
573
574 if (auto *GEP = dyn_cast<GEPOperator>(Val)) {
575 setInsertionPoint(Builder, GEP);
576 Value *Op = NewInsts[GEP->getOperand(0)];
577 Value *OffsetV = emitGEPOffset(&Builder, DL, GEP);
578 if (isa<ConstantInt>(Op) && cast<ConstantInt>(Op)->isZero())
579 NewInsts[GEP] = OffsetV;
580 else
581 NewInsts[GEP] = Builder.CreateAdd(
582 Op, OffsetV, GEP->getOperand(0)->getName() + ".add",
583 /*NUW=*/NW.hasNoUnsignedWrap(),
584 /*NSW=*/NW.hasNoUnsignedSignedWrap());
585 continue;
586 }
587 if (isa<PHINode>(Val))
588 continue;
589
590 llvm_unreachable("Unexpected instruction type");
591 }
592
593 // Add the incoming values to the PHI nodes.
594 for (Value *Val : Explored) {
595 if (Val == Base)
596 continue;
597 // All the instructions have been created, we can now add edges to the
598 // phi nodes.
599 if (auto *PHI = dyn_cast<PHINode>(Val)) {
600 PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]);
601 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) {
602 Value *NewIncoming = PHI->getIncomingValue(I);
603
604 auto It = NewInsts.find(NewIncoming);
605 if (It != NewInsts.end())
606 NewIncoming = It->second;
607
608 NewPhi->addIncoming(NewIncoming, PHI->getIncomingBlock(I));
609 }
610 }
611 }
612
613 for (Value *Val : Explored) {
614 if (Val == Base)
615 continue;
616
617 setInsertionPoint(Builder, Val, false);
618 // Create GEP for external users.
619 Value *NewVal = Builder.CreateGEP(Builder.getInt8Ty(), Base, NewInsts[Val],
620 Val->getName() + ".ptr", NW);
621 IC.replaceInstUsesWith(*cast<Instruction>(Val), NewVal);
622 // Add old instruction to worklist for DCE. We don't directly remove it
623 // here because the original compare is one of the users.
624 IC.addToWorklist(cast<Instruction>(Val));
625 }
626
627 return NewInsts[Start];
628}
629
630/// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant.
631/// We can look through PHIs, GEPs and casts in order to determine a common base
632/// between GEPLHS and RHS.
635 const DataLayout &DL,
636 InstCombiner &IC) {
637 // FIXME: Support vector of pointers.
638 if (GEPLHS->getType()->isVectorTy())
639 return nullptr;
640
641 if (!GEPLHS->hasAllConstantIndices())
642 return nullptr;
643
644 APInt Offset(DL.getIndexTypeSizeInBits(GEPLHS->getType()), 0);
645 Value *PtrBase =
647 /*AllowNonInbounds*/ false);
648
649 // Bail if we looked through addrspacecast.
650 if (PtrBase->getType() != GEPLHS->getType())
651 return nullptr;
652
653 // The set of nodes that will take part in this transformation.
654 SetVector<Value *> Nodes;
655 GEPNoWrapFlags NW = GEPLHS->getNoWrapFlags();
656 if (!canRewriteGEPAsOffset(RHS, PtrBase, NW, DL, Nodes))
657 return nullptr;
658
659 // We know we can re-write this as
660 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2)
661 // Since we've only looked through inbouds GEPs we know that we
662 // can't have overflow on either side. We can therefore re-write
663 // this as:
664 // OFFSET1 cmp OFFSET2
665 Value *NewRHS = rewriteGEPAsOffset(RHS, PtrBase, NW, DL, Nodes, IC);
666
667 // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written
668 // GEP having PtrBase as the pointer base, and has returned in NewRHS the
669 // offset. Since Index is the offset of LHS to the base pointer, we will now
670 // compare the offsets instead of comparing the pointers.
672 IC.Builder.getInt(Offset), NewRHS);
673}
674
675/// Fold comparisons between a GEP instruction and something else. At this point
676/// we know that the GEP is on the LHS of the comparison.
679 // Don't transform signed compares of GEPs into index compares. Even if the
680 // GEP is inbounds, the final add of the base pointer can have signed overflow
681 // and would change the result of the icmp.
682 // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be
683 // the maximum signed value for the pointer type.
685 return nullptr;
686
687 // Look through bitcasts and addrspacecasts. We do not however want to remove
688 // 0 GEPs.
689 if (!isa<GetElementPtrInst>(RHS))
691
692 auto CanFold = [Cond](GEPNoWrapFlags NW) {
694 return true;
695
696 // Unsigned predicates can be folded if the GEPs have *any* nowrap flags.
698 return NW != GEPNoWrapFlags::none();
699 };
700
701 auto NewICmp = [Cond](GEPNoWrapFlags NW, Value *Op1, Value *Op2) {
702 if (!NW.hasNoUnsignedWrap()) {
703 // Convert signed to unsigned comparison.
704 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Op1, Op2);
705 }
706
707 auto *I = new ICmpInst(Cond, Op1, Op2);
708 I->setSameSign(NW.hasNoUnsignedSignedWrap());
709 return I;
710 };
711
712 Value *PtrBase = GEPLHS->getOperand(0);
713 if (PtrBase == RHS && CanFold(GEPLHS->getNoWrapFlags())) {
714 // ((gep Ptr, OFFSET) cmp Ptr) ---> (OFFSET cmp 0).
715 Value *Offset = EmitGEPOffset(GEPLHS);
716 return NewICmp(GEPLHS->getNoWrapFlags(), Offset,
717 Constant::getNullValue(Offset->getType()));
718 }
719
720 if (GEPLHS->isInBounds() && ICmpInst::isEquality(Cond) &&
721 isa<Constant>(RHS) && cast<Constant>(RHS)->isNullValue() &&
722 !NullPointerIsDefined(I.getFunction(),
724 // For most address spaces, an allocation can't be placed at null, but null
725 // itself is treated as a 0 size allocation in the in bounds rules. Thus,
726 // the only valid inbounds address derived from null, is null itself.
727 // Thus, we have four cases to consider:
728 // 1) Base == nullptr, Offset == 0 -> inbounds, null
729 // 2) Base == nullptr, Offset != 0 -> poison as the result is out of bounds
730 // 3) Base != nullptr, Offset == (-base) -> poison (crossing allocations)
731 // 4) Base != nullptr, Offset != (-base) -> nonnull (and possibly poison)
732 //
733 // (Note if we're indexing a type of size 0, that simply collapses into one
734 // of the buckets above.)
735 //
736 // In general, we're allowed to make values less poison (i.e. remove
737 // sources of full UB), so in this case, we just select between the two
738 // non-poison cases (1 and 4 above).
739 //
740 // For vectors, we apply the same reasoning on a per-lane basis.
741 auto *Base = GEPLHS->getPointerOperand();
742 if (GEPLHS->getType()->isVectorTy() && Base->getType()->isPointerTy()) {
743 auto EC = cast<VectorType>(GEPLHS->getType())->getElementCount();
745 }
746 return new ICmpInst(Cond, Base,
748 cast<Constant>(RHS), Base->getType()));
749 } else if (GEPOperator *GEPRHS = dyn_cast<GEPOperator>(RHS)) {
750 // If the base pointers are different, but the indices are the same, just
751 // compare the base pointer.
752 if (PtrBase != GEPRHS->getOperand(0)) {
753 bool IndicesTheSame =
754 GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
755 GEPLHS->getPointerOperand()->getType() ==
756 GEPRHS->getPointerOperand()->getType() &&
757 GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType();
758 if (IndicesTheSame)
759 for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i)
760 if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) {
761 IndicesTheSame = false;
762 break;
763 }
764
765 // If all indices are the same, just compare the base pointers.
766 Type *BaseType = GEPLHS->getOperand(0)->getType();
767 if (IndicesTheSame && CmpInst::makeCmpResultType(BaseType) == I.getType())
768 return new ICmpInst(Cond, GEPLHS->getOperand(0), GEPRHS->getOperand(0));
769
770 // If we're comparing GEPs with two base pointers that only differ in type
771 // and both GEPs have only constant indices or just one use, then fold
772 // the compare with the adjusted indices.
773 // FIXME: Support vector of pointers.
774 if (GEPLHS->isInBounds() && GEPRHS->isInBounds() &&
775 (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) &&
776 (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) &&
777 PtrBase->stripPointerCasts() ==
778 GEPRHS->getOperand(0)->stripPointerCasts() &&
779 !GEPLHS->getType()->isVectorTy()) {
780 Value *LOffset = EmitGEPOffset(GEPLHS);
781 Value *ROffset = EmitGEPOffset(GEPRHS);
782
783 // If we looked through an addrspacecast between different sized address
784 // spaces, the LHS and RHS pointers are different sized
785 // integers. Truncate to the smaller one.
786 Type *LHSIndexTy = LOffset->getType();
787 Type *RHSIndexTy = ROffset->getType();
788 if (LHSIndexTy != RHSIndexTy) {
789 if (LHSIndexTy->getPrimitiveSizeInBits().getFixedValue() <
790 RHSIndexTy->getPrimitiveSizeInBits().getFixedValue()) {
791 ROffset = Builder.CreateTrunc(ROffset, LHSIndexTy);
792 } else
793 LOffset = Builder.CreateTrunc(LOffset, RHSIndexTy);
794 }
795
797 LOffset, ROffset);
798 return replaceInstUsesWith(I, Cmp);
799 }
800
801 // Otherwise, the base pointers are different and the indices are
802 // different. Try convert this to an indexed compare by looking through
803 // PHIs/casts.
804 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL, *this);
805 }
806
807 GEPNoWrapFlags NW = GEPLHS->getNoWrapFlags() & GEPRHS->getNoWrapFlags();
808 if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
809 GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType()) {
810 // If the GEPs only differ by one index, compare it.
811 unsigned NumDifferences = 0; // Keep track of # differences.
812 unsigned DiffOperand = 0; // The operand that differs.
813 for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i)
814 if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) {
815 Type *LHSType = GEPLHS->getOperand(i)->getType();
816 Type *RHSType = GEPRHS->getOperand(i)->getType();
817 // FIXME: Better support for vector of pointers.
818 if (LHSType->getPrimitiveSizeInBits() !=
819 RHSType->getPrimitiveSizeInBits() ||
820 (GEPLHS->getType()->isVectorTy() &&
821 (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) {
822 // Irreconcilable differences.
823 NumDifferences = 2;
824 break;
825 }
826
827 if (NumDifferences++) break;
828 DiffOperand = i;
829 }
830
831 if (NumDifferences == 0) // SAME GEP?
832 return replaceInstUsesWith(I, // No comparison is needed here.
833 ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond)));
834
835 else if (NumDifferences == 1 && CanFold(NW)) {
836 Value *LHSV = GEPLHS->getOperand(DiffOperand);
837 Value *RHSV = GEPRHS->getOperand(DiffOperand);
838 return NewICmp(NW, LHSV, RHSV);
839 }
840 }
841
842 if (CanFold(NW)) {
843 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2)
844 Value *L = EmitGEPOffset(GEPLHS, /*RewriteGEP=*/true);
845 Value *R = EmitGEPOffset(GEPRHS, /*RewriteGEP=*/true);
846 return NewICmp(NW, L, R);
847 }
848 }
849
850 // Try convert this to an indexed compare by looking through PHIs/casts as a
851 // last resort.
852 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL, *this);
853}
854
856 // It would be tempting to fold away comparisons between allocas and any
857 // pointer not based on that alloca (e.g. an argument). However, even
858 // though such pointers cannot alias, they can still compare equal.
859 //
860 // But LLVM doesn't specify where allocas get their memory, so if the alloca
861 // doesn't escape we can argue that it's impossible to guess its value, and we
862 // can therefore act as if any such guesses are wrong.
863 //
864 // However, we need to ensure that this folding is consistent: We can't fold
865 // one comparison to false, and then leave a different comparison against the
866 // same value alone (as it might evaluate to true at runtime, leading to a
867 // contradiction). As such, this code ensures that all comparisons are folded
868 // at the same time, and there are no other escapes.
869
870 struct CmpCaptureTracker : public CaptureTracker {
871 AllocaInst *Alloca;
872 bool Captured = false;
873 /// The value of the map is a bit mask of which icmp operands the alloca is
874 /// used in.
876
877 CmpCaptureTracker(AllocaInst *Alloca) : Alloca(Alloca) {}
878
879 void tooManyUses() override { Captured = true; }
880
881 bool captured(const Use *U) override {
882 auto *ICmp = dyn_cast<ICmpInst>(U->getUser());
883 // We need to check that U is based *only* on the alloca, and doesn't
884 // have other contributions from a select/phi operand.
885 // TODO: We could check whether getUnderlyingObjects() reduces to one
886 // object, which would allow looking through phi nodes.
887 if (ICmp && ICmp->isEquality() && getUnderlyingObject(*U) == Alloca) {
888 // Collect equality icmps of the alloca, and don't treat them as
889 // captures.
890 ICmps[ICmp] |= 1u << U->getOperandNo();
891 return false;
892 }
893
894 Captured = true;
895 return true;
896 }
897 };
898
899 CmpCaptureTracker Tracker(Alloca);
900 PointerMayBeCaptured(Alloca, &Tracker);
901 if (Tracker.Captured)
902 return false;
903
904 bool Changed = false;
905 for (auto [ICmp, Operands] : Tracker.ICmps) {
906 switch (Operands) {
907 case 1:
908 case 2: {
909 // The alloca is only used in one icmp operand. Assume that the
910 // equality is false.
911 auto *Res = ConstantInt::get(
912 ICmp->getType(), ICmp->getPredicate() == ICmpInst::ICMP_NE);
913 replaceInstUsesWith(*ICmp, Res);
915 Changed = true;
916 break;
917 }
918 case 3:
919 // Both icmp operands are based on the alloca, so this is comparing
920 // pointer offsets, without leaking any information about the address
921 // of the alloca. Ignore such comparisons.
922 break;
923 default:
924 llvm_unreachable("Cannot happen");
925 }
926 }
927
928 return Changed;
929}
930
931/// Fold "icmp pred (X+C), X".
933 CmpPredicate Pred) {
934 // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0,
935 // so the values can never be equal. Similarly for all other "or equals"
936 // operators.
937 assert(!!C && "C should not be zero!");
938
939 // (X+1) <u X --> X >u (MAXUINT-1) --> X == 255
940 // (X+2) <u X --> X >u (MAXUINT-2) --> X > 253
941 // (X+MAXUINT) <u X --> X >u (MAXUINT-MAXUINT) --> X != 0
942 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
943 Constant *R = ConstantInt::get(X->getType(),
944 APInt::getMaxValue(C.getBitWidth()) - C);
945 return new ICmpInst(ICmpInst::ICMP_UGT, X, R);
946 }
947
948 // (X+1) >u X --> X <u (0-1) --> X != 255
949 // (X+2) >u X --> X <u (0-2) --> X <u 254
950 // (X+MAXUINT) >u X --> X <u (0-MAXUINT) --> X <u 1 --> X == 0
951 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
952 return new ICmpInst(ICmpInst::ICMP_ULT, X,
953 ConstantInt::get(X->getType(), -C));
954
955 APInt SMax = APInt::getSignedMaxValue(C.getBitWidth());
956
957 // (X+ 1) <s X --> X >s (MAXSINT-1) --> X == 127
958 // (X+ 2) <s X --> X >s (MAXSINT-2) --> X >s 125
959 // (X+MAXSINT) <s X --> X >s (MAXSINT-MAXSINT) --> X >s 0
960 // (X+MINSINT) <s X --> X >s (MAXSINT-MINSINT) --> X >s -1
961 // (X+ -2) <s X --> X >s (MAXSINT- -2) --> X >s 126
962 // (X+ -1) <s X --> X >s (MAXSINT- -1) --> X != 127
963 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
964 return new ICmpInst(ICmpInst::ICMP_SGT, X,
965 ConstantInt::get(X->getType(), SMax - C));
966
967 // (X+ 1) >s X --> X <s (MAXSINT-(1-1)) --> X != 127
968 // (X+ 2) >s X --> X <s (MAXSINT-(2-1)) --> X <s 126
969 // (X+MAXSINT) >s X --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1
970 // (X+MINSINT) >s X --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2
971 // (X+ -2) >s X --> X <s (MAXSINT-(-2-1)) --> X <s -126
972 // (X+ -1) >s X --> X <s (MAXSINT-(-1-1)) --> X == -128
973
974 assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE);
975 return new ICmpInst(ICmpInst::ICMP_SLT, X,
976 ConstantInt::get(X->getType(), SMax - (C - 1)));
977}
978
979/// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" ->
980/// (icmp eq/ne A, Log2(AP2/AP1)) ->
981/// (icmp eq/ne A, Log2(AP2) - Log2(AP1)).
983 const APInt &AP1,
984 const APInt &AP2) {
985 assert(I.isEquality() && "Cannot fold icmp gt/lt");
986
987 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
988 if (I.getPredicate() == I.ICMP_NE)
989 Pred = CmpInst::getInversePredicate(Pred);
990 return new ICmpInst(Pred, LHS, RHS);
991 };
992
993 // Don't bother doing any work for cases which InstSimplify handles.
994 if (AP2.isZero())
995 return nullptr;
996
997 bool IsAShr = isa<AShrOperator>(I.getOperand(0));
998 if (IsAShr) {
999 if (AP2.isAllOnes())
1000 return nullptr;
1001 if (AP2.isNegative() != AP1.isNegative())
1002 return nullptr;
1003 if (AP2.sgt(AP1))
1004 return nullptr;
1005 }
1006
1007 if (!AP1)
1008 // 'A' must be large enough to shift out the highest set bit.
1009 return getICmp(I.ICMP_UGT, A,
1010 ConstantInt::get(A->getType(), AP2.logBase2()));
1011
1012 if (AP1 == AP2)
1013 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType()));
1014
1015 int Shift;
1016 if (IsAShr && AP1.isNegative())
1017 Shift = AP1.countl_one() - AP2.countl_one();
1018 else
1019 Shift = AP1.countl_zero() - AP2.countl_zero();
1020
1021 if (Shift > 0) {
1022 if (IsAShr && AP1 == AP2.ashr(Shift)) {
1023 // There are multiple solutions if we are comparing against -1 and the LHS
1024 // of the ashr is not a power of two.
1025 if (AP1.isAllOnes() && !AP2.isPowerOf2())
1026 return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift));
1027 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1028 } else if (AP1 == AP2.lshr(Shift)) {
1029 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1030 }
1031 }
1032
1033 // Shifting const2 will never be equal to const1.
1034 // FIXME: This should always be handled by InstSimplify?
1035 auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE);
1036 return replaceInstUsesWith(I, TorF);
1037}
1038
1039/// Handle "(icmp eq/ne (shl AP2, A), AP1)" ->
1040/// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)).
1042 const APInt &AP1,
1043 const APInt &AP2) {
1044 assert(I.isEquality() && "Cannot fold icmp gt/lt");
1045
1046 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
1047 if (I.getPredicate() == I.ICMP_NE)
1048 Pred = CmpInst::getInversePredicate(Pred);
1049 return new ICmpInst(Pred, LHS, RHS);
1050 };
1051
1052 // Don't bother doing any work for cases which InstSimplify handles.
1053 if (AP2.isZero())
1054 return nullptr;
1055
1056 unsigned AP2TrailingZeros = AP2.countr_zero();
1057
1058 if (!AP1 && AP2TrailingZeros != 0)
1059 return getICmp(
1060 I.ICMP_UGE, A,
1061 ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros));
1062
1063 if (AP1 == AP2)
1064 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType()));
1065
1066 // Get the distance between the lowest bits that are set.
1067 int Shift = AP1.countr_zero() - AP2TrailingZeros;
1068
1069 if (Shift > 0 && AP2.shl(Shift) == AP1)
1070 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1071
1072 // Shifting const2 will never be equal to const1.
1073 // FIXME: This should always be handled by InstSimplify?
1074 auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE);
1075 return replaceInstUsesWith(I, TorF);
1076}
1077
1078/// The caller has matched a pattern of the form:
1079/// I = icmp ugt (add (add A, B), CI2), CI1
1080/// If this is of the form:
1081/// sum = a + b
1082/// if (sum+128 >u 255)
1083/// Then replace it with llvm.sadd.with.overflow.i8.
1084///
1086 ConstantInt *CI2, ConstantInt *CI1,
1087 InstCombinerImpl &IC) {
1088 // The transformation we're trying to do here is to transform this into an
1089 // llvm.sadd.with.overflow. To do this, we have to replace the original add
1090 // with a narrower add, and discard the add-with-constant that is part of the
1091 // range check (if we can't eliminate it, this isn't profitable).
1092
1093 // In order to eliminate the add-with-constant, the compare can be its only
1094 // use.
1095 Instruction *AddWithCst = cast<Instruction>(I.getOperand(0));
1096 if (!AddWithCst->hasOneUse())
1097 return nullptr;
1098
1099 // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow.
1100 if (!CI2->getValue().isPowerOf2())
1101 return nullptr;
1102 unsigned NewWidth = CI2->getValue().countr_zero();
1103 if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31)
1104 return nullptr;
1105
1106 // The width of the new add formed is 1 more than the bias.
1107 ++NewWidth;
1108
1109 // Check to see that CI1 is an all-ones value with NewWidth bits.
1110 if (CI1->getBitWidth() == NewWidth ||
1111 CI1->getValue() != APInt::getLowBitsSet(CI1->getBitWidth(), NewWidth))
1112 return nullptr;
1113
1114 // This is only really a signed overflow check if the inputs have been
1115 // sign-extended; check for that condition. For example, if CI2 is 2^31 and
1116 // the operands of the add are 64 bits wide, we need at least 33 sign bits.
1117 if (IC.ComputeMaxSignificantBits(A, 0, &I) > NewWidth ||
1118 IC.ComputeMaxSignificantBits(B, 0, &I) > NewWidth)
1119 return nullptr;
1120
1121 // In order to replace the original add with a narrower
1122 // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant
1123 // and truncates that discard the high bits of the add. Verify that this is
1124 // the case.
1125 Instruction *OrigAdd = cast<Instruction>(AddWithCst->getOperand(0));
1126 for (User *U : OrigAdd->users()) {
1127 if (U == AddWithCst)
1128 continue;
1129
1130 // Only accept truncates for now. We would really like a nice recursive
1131 // predicate like SimplifyDemandedBits, but which goes downwards the use-def
1132 // chain to see which bits of a value are actually demanded. If the
1133 // original add had another add which was then immediately truncated, we
1134 // could still do the transformation.
1135 TruncInst *TI = dyn_cast<TruncInst>(U);
1136 if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth)
1137 return nullptr;
1138 }
1139
1140 // If the pattern matches, truncate the inputs to the narrower type and
1141 // use the sadd_with_overflow intrinsic to efficiently compute both the
1142 // result and the overflow bit.
1143 Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth);
1145 I.getModule(), Intrinsic::sadd_with_overflow, NewType);
1146
1147 InstCombiner::BuilderTy &Builder = IC.Builder;
1148
1149 // Put the new code above the original add, in case there are any uses of the
1150 // add between the add and the compare.
1151 Builder.SetInsertPoint(OrigAdd);
1152
1153 Value *TruncA = Builder.CreateTrunc(A, NewType, A->getName() + ".trunc");
1154 Value *TruncB = Builder.CreateTrunc(B, NewType, B->getName() + ".trunc");
1155 CallInst *Call = Builder.CreateCall(F, {TruncA, TruncB}, "sadd");
1156 Value *Add = Builder.CreateExtractValue(Call, 0, "sadd.result");
1157 Value *ZExt = Builder.CreateZExt(Add, OrigAdd->getType());
1158
1159 // The inner add was the result of the narrow add, zero extended to the
1160 // wider type. Replace it with the result computed by the intrinsic.
1161 IC.replaceInstUsesWith(*OrigAdd, ZExt);
1162 IC.eraseInstFromFunction(*OrigAdd);
1163
1164 // The original icmp gets replaced with the overflow value.
1165 return ExtractValueInst::Create(Call, 1, "sadd.overflow");
1166}
1167
1168/// If we have:
1169/// icmp eq/ne (urem/srem %x, %y), 0
1170/// iff %y is a power-of-two, we can replace this with a bit test:
1171/// icmp eq/ne (and %x, (add %y, -1)), 0
1173 // This fold is only valid for equality predicates.
1174 if (!I.isEquality())
1175 return nullptr;
1176 CmpPredicate Pred;
1177 Value *X, *Y, *Zero;
1178 if (!match(&I, m_ICmp(Pred, m_OneUse(m_IRem(m_Value(X), m_Value(Y))),
1179 m_CombineAnd(m_Zero(), m_Value(Zero)))))
1180 return nullptr;
1181 if (!isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, 0, &I))
1182 return nullptr;
1183 // This may increase instruction count, we don't enforce that Y is a constant.
1184 Value *Mask = Builder.CreateAdd(Y, Constant::getAllOnesValue(Y->getType()));
1185 Value *Masked = Builder.CreateAnd(X, Mask);
1186 return ICmpInst::Create(Instruction::ICmp, Pred, Masked, Zero);
1187}
1188
1189/// Fold equality-comparison between zero and any (maybe truncated) right-shift
1190/// by one-less-than-bitwidth into a sign test on the original value.
1192 Instruction *Val;
1193 CmpPredicate Pred;
1194 if (!I.isEquality() || !match(&I, m_ICmp(Pred, m_Instruction(Val), m_Zero())))
1195 return nullptr;
1196
1197 Value *X;
1198 Type *XTy;
1199
1200 Constant *C;
1201 if (match(Val, m_TruncOrSelf(m_Shr(m_Value(X), m_Constant(C))))) {
1202 XTy = X->getType();
1203 unsigned XBitWidth = XTy->getScalarSizeInBits();
1205 APInt(XBitWidth, XBitWidth - 1))))
1206 return nullptr;
1207 } else if (isa<BinaryOperator>(Val) &&
1209 cast<BinaryOperator>(Val), SQ.getWithInstruction(Val),
1210 /*AnalyzeForSignBitExtraction=*/true))) {
1211 XTy = X->getType();
1212 } else
1213 return nullptr;
1214
1215 return ICmpInst::Create(Instruction::ICmp,
1219}
1220
1221// Handle icmp pred X, 0
1223 CmpInst::Predicate Pred = Cmp.getPredicate();
1224 if (!match(Cmp.getOperand(1), m_Zero()))
1225 return nullptr;
1226
1227 // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0)
1228 if (Pred == ICmpInst::ICMP_SGT) {
1229 Value *A, *B;
1230 if (match(Cmp.getOperand(0), m_SMin(m_Value(A), m_Value(B)))) {
1232 return new ICmpInst(Pred, B, Cmp.getOperand(1));
1234 return new ICmpInst(Pred, A, Cmp.getOperand(1));
1235 }
1236 }
1237
1239 return New;
1240
1241 // Given:
1242 // icmp eq/ne (urem %x, %y), 0
1243 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
1244 // icmp eq/ne %x, 0
1245 Value *X, *Y;
1246 if (match(Cmp.getOperand(0), m_URem(m_Value(X), m_Value(Y))) &&
1247 ICmpInst::isEquality(Pred)) {
1248 KnownBits XKnown = computeKnownBits(X, 0, &Cmp);
1249 KnownBits YKnown = computeKnownBits(Y, 0, &Cmp);
1250 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
1251 return new ICmpInst(Pred, X, Cmp.getOperand(1));
1252 }
1253
1254 // (icmp eq/ne (mul X Y)) -> (icmp eq/ne X/Y) if we know about whether X/Y are
1255 // odd/non-zero/there is no overflow.
1256 if (match(Cmp.getOperand(0), m_Mul(m_Value(X), m_Value(Y))) &&
1257 ICmpInst::isEquality(Pred)) {
1258
1259 KnownBits XKnown = computeKnownBits(X, 0, &Cmp);
1260 // if X % 2 != 0
1261 // (icmp eq/ne Y)
1262 if (XKnown.countMaxTrailingZeros() == 0)
1263 return new ICmpInst(Pred, Y, Cmp.getOperand(1));
1264
1265 KnownBits YKnown = computeKnownBits(Y, 0, &Cmp);
1266 // if Y % 2 != 0
1267 // (icmp eq/ne X)
1268 if (YKnown.countMaxTrailingZeros() == 0)
1269 return new ICmpInst(Pred, X, Cmp.getOperand(1));
1270
1271 auto *BO0 = cast<OverflowingBinaryOperator>(Cmp.getOperand(0));
1272 if (BO0->hasNoUnsignedWrap() || BO0->hasNoSignedWrap()) {
1273 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
1274 // `isKnownNonZero` does more analysis than just `!KnownBits.One.isZero()`
1275 // but to avoid unnecessary work, first just if this is an obvious case.
1276
1277 // if X non-zero and NoOverflow(X * Y)
1278 // (icmp eq/ne Y)
1279 if (!XKnown.One.isZero() || isKnownNonZero(X, Q))
1280 return new ICmpInst(Pred, Y, Cmp.getOperand(1));
1281
1282 // if Y non-zero and NoOverflow(X * Y)
1283 // (icmp eq/ne X)
1284 if (!YKnown.One.isZero() || isKnownNonZero(Y, Q))
1285 return new ICmpInst(Pred, X, Cmp.getOperand(1));
1286 }
1287 // Note, we are skipping cases:
1288 // if Y % 2 != 0 AND X % 2 != 0
1289 // (false/true)
1290 // if X non-zero and Y non-zero and NoOverflow(X * Y)
1291 // (false/true)
1292 // Those can be simplified later as we would have already replaced the (icmp
1293 // eq/ne (mul X, Y)) with (icmp eq/ne X/Y) and if X/Y is known non-zero that
1294 // will fold to a constant elsewhere.
1295 }
1296 return nullptr;
1297}
1298
1299/// Fold icmp Pred X, C.
1300/// TODO: This code structure does not make sense. The saturating add fold
1301/// should be moved to some other helper and extended as noted below (it is also
1302/// possible that code has been made unnecessary - do we canonicalize IR to
1303/// overflow/saturating intrinsics or not?).
1305 // Match the following pattern, which is a common idiom when writing
1306 // overflow-safe integer arithmetic functions. The source performs an addition
1307 // in wider type and explicitly checks for overflow using comparisons against
1308 // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic.
1309 //
1310 // TODO: This could probably be generalized to handle other overflow-safe
1311 // operations if we worked out the formulas to compute the appropriate magic
1312 // constants.
1313 //
1314 // sum = a + b
1315 // if (sum+128 >u 255) ... -> llvm.sadd.with.overflow.i8
1316 CmpInst::Predicate Pred = Cmp.getPredicate();
1317 Value *Op0 = Cmp.getOperand(0), *Op1 = Cmp.getOperand(1);
1318 Value *A, *B;
1319 ConstantInt *CI, *CI2; // I = icmp ugt (add (add A, B), CI2), CI
1320 if (Pred == ICmpInst::ICMP_UGT && match(Op1, m_ConstantInt(CI)) &&
1321 match(Op0, m_Add(m_Add(m_Value(A), m_Value(B)), m_ConstantInt(CI2))))
1322 if (Instruction *Res = processUGT_ADDCST_ADD(Cmp, A, B, CI2, CI, *this))
1323 return Res;
1324
1325 // icmp(phi(C1, C2, ...), C) -> phi(icmp(C1, C), icmp(C2, C), ...).
1326 Constant *C = dyn_cast<Constant>(Op1);
1327 if (!C)
1328 return nullptr;
1329
1330 if (auto *Phi = dyn_cast<PHINode>(Op0))
1331 if (all_of(Phi->operands(), [](Value *V) { return isa<Constant>(V); })) {
1333 for (Value *V : Phi->incoming_values()) {
1334 Constant *Res =
1335 ConstantFoldCompareInstOperands(Pred, cast<Constant>(V), C, DL);
1336 if (!Res)
1337 return nullptr;
1338 Ops.push_back(Res);
1339 }
1341 PHINode *NewPhi = Builder.CreatePHI(Cmp.getType(), Phi->getNumOperands());
1342 for (auto [V, Pred] : zip(Ops, Phi->blocks()))
1343 NewPhi->addIncoming(V, Pred);
1344 return replaceInstUsesWith(Cmp, NewPhi);
1345 }
1346
1348 return R;
1349
1350 return nullptr;
1351}
1352
1353/// Canonicalize icmp instructions based on dominating conditions.
1355 // We already checked simple implication in InstSimplify, only handle complex
1356 // cases here.
1357 Value *X = Cmp.getOperand(0), *Y = Cmp.getOperand(1);
1358 const APInt *C;
1359 if (!match(Y, m_APInt(C)))
1360 return nullptr;
1361
1362 CmpInst::Predicate Pred = Cmp.getPredicate();
1364
1365 auto handleDomCond = [&](ICmpInst::Predicate DomPred,
1366 const APInt *DomC) -> Instruction * {
1367 // We have 2 compares of a variable with constants. Calculate the constant
1368 // ranges of those compares to see if we can transform the 2nd compare:
1369 // DomBB:
1370 // DomCond = icmp DomPred X, DomC
1371 // br DomCond, CmpBB, FalseBB
1372 // CmpBB:
1373 // Cmp = icmp Pred X, C
1374 ConstantRange DominatingCR =
1375 ConstantRange::makeExactICmpRegion(DomPred, *DomC);
1376 ConstantRange Intersection = DominatingCR.intersectWith(CR);
1377 ConstantRange Difference = DominatingCR.difference(CR);
1378 if (Intersection.isEmptySet())
1379 return replaceInstUsesWith(Cmp, Builder.getFalse());
1380 if (Difference.isEmptySet())
1381 return replaceInstUsesWith(Cmp, Builder.getTrue());
1382
1383 // Canonicalizing a sign bit comparison that gets used in a branch,
1384 // pessimizes codegen by generating branch on zero instruction instead
1385 // of a test and branch. So we avoid canonicalizing in such situations
1386 // because test and branch instruction has better branch displacement
1387 // than compare and branch instruction.
1388 bool UnusedBit;
1389 bool IsSignBit = isSignBitCheck(Pred, *C, UnusedBit);
1390 if (Cmp.isEquality() || (IsSignBit && hasBranchUse(Cmp)))
1391 return nullptr;
1392
1393 // Avoid an infinite loop with min/max canonicalization.
1394 // TODO: This will be unnecessary if we canonicalize to min/max intrinsics.
1395 if (Cmp.hasOneUse() &&
1396 match(Cmp.user_back(), m_MaxOrMin(m_Value(), m_Value())))
1397 return nullptr;
1398
1399 if (const APInt *EqC = Intersection.getSingleElement())
1400 return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(*EqC));
1401 if (const APInt *NeC = Difference.getSingleElement())
1402 return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(*NeC));
1403 return nullptr;
1404 };
1405
1406 for (BranchInst *BI : DC.conditionsFor(X)) {
1407 CmpPredicate DomPred;
1408 const APInt *DomC;
1409 if (!match(BI->getCondition(),
1410 m_ICmp(DomPred, m_Specific(X), m_APInt(DomC))))
1411 continue;
1412
1413 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0));
1414 if (DT.dominates(Edge0, Cmp.getParent())) {
1415 if (auto *V = handleDomCond(DomPred, DomC))
1416 return V;
1417 } else {
1418 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1));
1419 if (DT.dominates(Edge1, Cmp.getParent()))
1420 if (auto *V =
1421 handleDomCond(CmpInst::getInversePredicate(DomPred), DomC))
1422 return V;
1423 }
1424 }
1425
1426 return nullptr;
1427}
1428
1429/// Fold icmp (trunc X), C.
1431 TruncInst *Trunc,
1432 const APInt &C) {
1433 ICmpInst::Predicate Pred = Cmp.getPredicate();
1434 Value *X = Trunc->getOperand(0);
1435 Type *SrcTy = X->getType();
1436 unsigned DstBits = Trunc->getType()->getScalarSizeInBits(),
1437 SrcBits = SrcTy->getScalarSizeInBits();
1438
1439 // Match (icmp pred (trunc nuw/nsw X), C)
1440 // Which we can convert to (icmp pred X, (sext/zext C))
1441 if (shouldChangeType(Trunc->getType(), SrcTy)) {
1442 if (Trunc->hasNoSignedWrap())
1443 return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, C.sext(SrcBits)));
1444 if (!Cmp.isSigned() && Trunc->hasNoUnsignedWrap())
1445 return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, C.zext(SrcBits)));
1446 }
1447
1448 if (C.isOne() && C.getBitWidth() > 1) {
1449 // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1
1450 Value *V = nullptr;
1451 if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V))))
1452 return new ICmpInst(ICmpInst::ICMP_SLT, V,
1453 ConstantInt::get(V->getType(), 1));
1454 }
1455
1456 // TODO: Handle any shifted constant by subtracting trailing zeros.
1457 // TODO: Handle non-equality predicates.
1458 Value *Y;
1459 if (Cmp.isEquality() && match(X, m_Shl(m_One(), m_Value(Y)))) {
1460 // (trunc (1 << Y) to iN) == 0 --> Y u>= N
1461 // (trunc (1 << Y) to iN) != 0 --> Y u< N
1462 if (C.isZero()) {
1463 auto NewPred = (Pred == Cmp.ICMP_EQ) ? Cmp.ICMP_UGE : Cmp.ICMP_ULT;
1464 return new ICmpInst(NewPred, Y, ConstantInt::get(SrcTy, DstBits));
1465 }
1466 // (trunc (1 << Y) to iN) == 2**C --> Y == C
1467 // (trunc (1 << Y) to iN) != 2**C --> Y != C
1468 if (C.isPowerOf2())
1469 return new ICmpInst(Pred, Y, ConstantInt::get(SrcTy, C.logBase2()));
1470 }
1471
1472 if (Cmp.isEquality() && Trunc->hasOneUse()) {
1473 // Canonicalize to a mask and wider compare if the wide type is suitable:
1474 // (trunc X to i8) == C --> (X & 0xff) == (zext C)
1475 if (!SrcTy->isVectorTy() && shouldChangeType(DstBits, SrcBits)) {
1476 Constant *Mask =
1477 ConstantInt::get(SrcTy, APInt::getLowBitsSet(SrcBits, DstBits));
1478 Value *And = Builder.CreateAnd(X, Mask);
1479 Constant *WideC = ConstantInt::get(SrcTy, C.zext(SrcBits));
1480 return new ICmpInst(Pred, And, WideC);
1481 }
1482
1483 // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all
1484 // of the high bits truncated out of x are known.
1485 KnownBits Known = computeKnownBits(X, 0, &Cmp);
1486
1487 // If all the high bits are known, we can do this xform.
1488 if ((Known.Zero | Known.One).countl_one() >= SrcBits - DstBits) {
1489 // Pull in the high bits from known-ones set.
1490 APInt NewRHS = C.zext(SrcBits);
1491 NewRHS |= Known.One & APInt::getHighBitsSet(SrcBits, SrcBits - DstBits);
1492 return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, NewRHS));
1493 }
1494 }
1495
1496 // Look through truncated right-shift of the sign-bit for a sign-bit check:
1497 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] < 0 --> ShOp < 0
1498 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] > -1 --> ShOp > -1
1499 Value *ShOp;
1500 const APInt *ShAmtC;
1501 bool TrueIfSigned;
1502 if (isSignBitCheck(Pred, C, TrueIfSigned) &&
1503 match(X, m_Shr(m_Value(ShOp), m_APInt(ShAmtC))) &&
1504 DstBits == SrcBits - ShAmtC->getZExtValue()) {
1505 return TrueIfSigned ? new ICmpInst(ICmpInst::ICMP_SLT, ShOp,
1507 : new ICmpInst(ICmpInst::ICMP_SGT, ShOp,
1509 }
1510
1511 return nullptr;
1512}
1513
1514/// Fold icmp (trunc nuw/nsw X), (trunc nuw/nsw Y).
1515/// Fold icmp (trunc nuw/nsw X), (zext/sext Y).
1518 const SimplifyQuery &Q) {
1519 Value *X, *Y;
1520 CmpPredicate Pred;
1521 bool YIsSExt = false;
1522 // Try to match icmp (trunc X), (trunc Y)
1523 if (match(&Cmp, m_ICmp(Pred, m_Trunc(m_Value(X)), m_Trunc(m_Value(Y))))) {
1524 unsigned NoWrapFlags = cast<TruncInst>(Cmp.getOperand(0))->getNoWrapKind() &
1525 cast<TruncInst>(Cmp.getOperand(1))->getNoWrapKind();
1526 if (Cmp.isSigned()) {
1527 // For signed comparisons, both truncs must be nsw.
1528 if (!(NoWrapFlags & TruncInst::NoSignedWrap))
1529 return nullptr;
1530 } else {
1531 // For unsigned and equality comparisons, either both must be nuw or
1532 // both must be nsw, we don't care which.
1533 if (!NoWrapFlags)
1534 return nullptr;
1535 }
1536
1537 if (X->getType() != Y->getType() &&
1538 (!Cmp.getOperand(0)->hasOneUse() || !Cmp.getOperand(1)->hasOneUse()))
1539 return nullptr;
1540 if (!isDesirableIntType(X->getType()->getScalarSizeInBits()) &&
1541 isDesirableIntType(Y->getType()->getScalarSizeInBits())) {
1542 std::swap(X, Y);
1543 Pred = Cmp.getSwappedPredicate(Pred);
1544 }
1545 YIsSExt = !(NoWrapFlags & TruncInst::NoUnsignedWrap);
1546 }
1547 // Try to match icmp (trunc nuw X), (zext Y)
1548 else if (!Cmp.isSigned() &&
1549 match(&Cmp, m_c_ICmp(Pred, m_NUWTrunc(m_Value(X)),
1550 m_OneUse(m_ZExt(m_Value(Y)))))) {
1551 // Can fold trunc nuw + zext for unsigned and equality predicates.
1552 }
1553 // Try to match icmp (trunc nsw X), (sext Y)
1554 else if (match(&Cmp, m_c_ICmp(Pred, m_NSWTrunc(m_Value(X)),
1556 // Can fold trunc nsw + zext/sext for all predicates.
1557 YIsSExt =
1558 isa<SExtInst>(Cmp.getOperand(0)) || isa<SExtInst>(Cmp.getOperand(1));
1559 } else
1560 return nullptr;
1561
1562 Type *TruncTy = Cmp.getOperand(0)->getType();
1563 unsigned TruncBits = TruncTy->getScalarSizeInBits();
1564
1565 // If this transform will end up changing from desirable types -> undesirable
1566 // types skip it.
1567 if (isDesirableIntType(TruncBits) &&
1568 !isDesirableIntType(X->getType()->getScalarSizeInBits()))
1569 return nullptr;
1570
1571 Value *NewY = Builder.CreateIntCast(Y, X->getType(), YIsSExt);
1572 return new ICmpInst(Pred, X, NewY);
1573}
1574
1575/// Fold icmp (xor X, Y), C.
1578 const APInt &C) {
1579 if (Instruction *I = foldICmpXorShiftConst(Cmp, Xor, C))
1580 return I;
1581
1582 Value *X = Xor->getOperand(0);
1583 Value *Y = Xor->getOperand(1);
1584 const APInt *XorC;
1585 if (!match(Y, m_APInt(XorC)))
1586 return nullptr;
1587
1588 // If this is a comparison that tests the signbit (X < 0) or (x > -1),
1589 // fold the xor.
1590 ICmpInst::Predicate Pred = Cmp.getPredicate();
1591 bool TrueIfSigned = false;
1592 if (isSignBitCheck(Cmp.getPredicate(), C, TrueIfSigned)) {
1593
1594 // If the sign bit of the XorCst is not set, there is no change to
1595 // the operation, just stop using the Xor.
1596 if (!XorC->isNegative())
1597 return replaceOperand(Cmp, 0, X);
1598
1599 // Emit the opposite comparison.
1600 if (TrueIfSigned)
1601 return new ICmpInst(ICmpInst::ICMP_SGT, X,
1602 ConstantInt::getAllOnesValue(X->getType()));
1603 else
1604 return new ICmpInst(ICmpInst::ICMP_SLT, X,
1605 ConstantInt::getNullValue(X->getType()));
1606 }
1607
1608 if (Xor->hasOneUse()) {
1609 // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask))
1610 if (!Cmp.isEquality() && XorC->isSignMask()) {
1611 Pred = Cmp.getFlippedSignednessPredicate();
1612 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC));
1613 }
1614
1615 // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask))
1616 if (!Cmp.isEquality() && XorC->isMaxSignedValue()) {
1617 Pred = Cmp.getFlippedSignednessPredicate();
1618 Pred = Cmp.getSwappedPredicate(Pred);
1619 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC));
1620 }
1621 }
1622
1623 // Mask constant magic can eliminate an 'xor' with unsigned compares.
1624 if (Pred == ICmpInst::ICMP_UGT) {
1625 // (xor X, ~C) >u C --> X <u ~C (when C+1 is a power of 2)
1626 if (*XorC == ~C && (C + 1).isPowerOf2())
1627 return new ICmpInst(ICmpInst::ICMP_ULT, X, Y);
1628 // (xor X, C) >u C --> X >u C (when C+1 is a power of 2)
1629 if (*XorC == C && (C + 1).isPowerOf2())
1630 return new ICmpInst(ICmpInst::ICMP_UGT, X, Y);
1631 }
1632 if (Pred == ICmpInst::ICMP_ULT) {
1633 // (xor X, -C) <u C --> X >u ~C (when C is a power of 2)
1634 if (*XorC == -C && C.isPowerOf2())
1635 return new ICmpInst(ICmpInst::ICMP_UGT, X,
1636 ConstantInt::get(X->getType(), ~C));
1637 // (xor X, C) <u C --> X >u ~C (when -C is a power of 2)
1638 if (*XorC == C && (-C).isPowerOf2())
1639 return new ICmpInst(ICmpInst::ICMP_UGT, X,
1640 ConstantInt::get(X->getType(), ~C));
1641 }
1642 return nullptr;
1643}
1644
1645/// For power-of-2 C:
1646/// ((X s>> ShiftC) ^ X) u< C --> (X + C) u< (C << 1)
1647/// ((X s>> ShiftC) ^ X) u> (C - 1) --> (X + C) u> ((C << 1) - 1)
1650 const APInt &C) {
1651 CmpInst::Predicate Pred = Cmp.getPredicate();
1652 APInt PowerOf2;
1653 if (Pred == ICmpInst::ICMP_ULT)
1654 PowerOf2 = C;
1655 else if (Pred == ICmpInst::ICMP_UGT && !C.isMaxValue())
1656 PowerOf2 = C + 1;
1657 else
1658 return nullptr;
1659 if (!PowerOf2.isPowerOf2())
1660 return nullptr;
1661 Value *X;
1662 const APInt *ShiftC;
1664 m_AShr(m_Deferred(X), m_APInt(ShiftC))))))
1665 return nullptr;
1666 uint64_t Shift = ShiftC->getLimitedValue();
1667 Type *XType = X->getType();
1668 if (Shift == 0 || PowerOf2.isMinSignedValue())
1669 return nullptr;
1670 Value *Add = Builder.CreateAdd(X, ConstantInt::get(XType, PowerOf2));
1671 APInt Bound =
1672 Pred == ICmpInst::ICMP_ULT ? PowerOf2 << 1 : ((PowerOf2 << 1) - 1);
1673 return new ICmpInst(Pred, Add, ConstantInt::get(XType, Bound));
1674}
1675
1676/// Fold icmp (and (sh X, Y), C2), C1.
1679 const APInt &C1,
1680 const APInt &C2) {
1681 BinaryOperator *Shift = dyn_cast<BinaryOperator>(And->getOperand(0));
1682 if (!Shift || !Shift->isShift())
1683 return nullptr;
1684
1685 // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could
1686 // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in
1687 // code produced by the clang front-end, for bitfield access.
1688 // This seemingly simple opportunity to fold away a shift turns out to be
1689 // rather complicated. See PR17827 for details.
1690 unsigned ShiftOpcode = Shift->getOpcode();
1691 bool IsShl = ShiftOpcode == Instruction::Shl;
1692 const APInt *C3;
1693 if (match(Shift->getOperand(1), m_APInt(C3))) {
1694 APInt NewAndCst, NewCmpCst;
1695 bool AnyCmpCstBitsShiftedOut;
1696 if (ShiftOpcode == Instruction::Shl) {
1697 // For a left shift, we can fold if the comparison is not signed. We can
1698 // also fold a signed comparison if the mask value and comparison value
1699 // are not negative. These constraints may not be obvious, but we can
1700 // prove that they are correct using an SMT solver.
1701 if (Cmp.isSigned() && (C2.isNegative() || C1.isNegative()))
1702 return nullptr;
1703
1704 NewCmpCst = C1.lshr(*C3);
1705 NewAndCst = C2.lshr(*C3);
1706 AnyCmpCstBitsShiftedOut = NewCmpCst.shl(*C3) != C1;
1707 } else if (ShiftOpcode == Instruction::LShr) {
1708 // For a logical right shift, we can fold if the comparison is not signed.
1709 // We can also fold a signed comparison if the shifted mask value and the
1710 // shifted comparison value are not negative. These constraints may not be
1711 // obvious, but we can prove that they are correct using an SMT solver.
1712 NewCmpCst = C1.shl(*C3);
1713 NewAndCst = C2.shl(*C3);
1714 AnyCmpCstBitsShiftedOut = NewCmpCst.lshr(*C3) != C1;
1715 if (Cmp.isSigned() && (NewAndCst.isNegative() || NewCmpCst.isNegative()))
1716 return nullptr;
1717 } else {
1718 // For an arithmetic shift, check that both constants don't use (in a
1719 // signed sense) the top bits being shifted out.
1720 assert(ShiftOpcode == Instruction::AShr && "Unknown shift opcode");
1721 NewCmpCst = C1.shl(*C3);
1722 NewAndCst = C2.shl(*C3);
1723 AnyCmpCstBitsShiftedOut = NewCmpCst.ashr(*C3) != C1;
1724 if (NewAndCst.ashr(*C3) != C2)
1725 return nullptr;
1726 }
1727
1728 if (AnyCmpCstBitsShiftedOut) {
1729 // If we shifted bits out, the fold is not going to work out. As a
1730 // special case, check to see if this means that the result is always
1731 // true or false now.
1732 if (Cmp.getPredicate() == ICmpInst::ICMP_EQ)
1733 return replaceInstUsesWith(Cmp, ConstantInt::getFalse(Cmp.getType()));
1734 if (Cmp.getPredicate() == ICmpInst::ICMP_NE)
1735 return replaceInstUsesWith(Cmp, ConstantInt::getTrue(Cmp.getType()));
1736 } else {
1737 Value *NewAnd = Builder.CreateAnd(
1738 Shift->getOperand(0), ConstantInt::get(And->getType(), NewAndCst));
1739 return new ICmpInst(Cmp.getPredicate(),
1740 NewAnd, ConstantInt::get(And->getType(), NewCmpCst));
1741 }
1742 }
1743
1744 // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is
1745 // preferable because it allows the C2 << Y expression to be hoisted out of a
1746 // loop if Y is invariant and X is not.
1747 if (Shift->hasOneUse() && C1.isZero() && Cmp.isEquality() &&
1748 !Shift->isArithmeticShift() &&
1749 ((!IsShl && C2.isOne()) || !isa<Constant>(Shift->getOperand(0)))) {
1750 // Compute C2 << Y.
1751 Value *NewShift =
1752 IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1))
1753 : Builder.CreateShl(And->getOperand(1), Shift->getOperand(1));
1754
1755 // Compute X & (C2 << Y).
1756 Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift);
1757 return new ICmpInst(Cmp.getPredicate(), NewAnd, Cmp.getOperand(1));
1758 }
1759
1760 return nullptr;
1761}
1762
1763/// Fold icmp (and X, C2), C1.
1766 const APInt &C1) {
1767 bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE;
1768
1769 // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1
1770 // TODO: We canonicalize to the longer form for scalars because we have
1771 // better analysis/folds for icmp, and codegen may be better with icmp.
1772 if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isZero() &&
1773 match(And->getOperand(1), m_One()))
1774 return new TruncInst(And->getOperand(0), Cmp.getType());
1775
1776 const APInt *C2;
1777 Value *X;
1778 if (!match(And, m_And(m_Value(X), m_APInt(C2))))
1779 return nullptr;
1780
1781 // (and X, highmask) s> [0, ~highmask] --> X s> ~highmask
1782 if (Cmp.getPredicate() == ICmpInst::ICMP_SGT && C1.ule(~*C2) &&
1783 C2->isNegatedPowerOf2())
1784 return new ICmpInst(ICmpInst::ICMP_SGT, X,
1785 ConstantInt::get(X->getType(), ~*C2));
1786 // (and X, highmask) s< [1, -highmask] --> X s< -highmask
1787 if (Cmp.getPredicate() == ICmpInst::ICMP_SLT && !C1.isSignMask() &&
1788 (C1 - 1).ule(~*C2) && C2->isNegatedPowerOf2() && !C2->isSignMask())
1789 return new ICmpInst(ICmpInst::ICMP_SLT, X,
1790 ConstantInt::get(X->getType(), -*C2));
1791
1792 // Don't perform the following transforms if the AND has multiple uses
1793 if (!And->hasOneUse())
1794 return nullptr;
1795
1796 if (Cmp.isEquality() && C1.isZero()) {
1797 // Restrict this fold to single-use 'and' (PR10267).
1798 // Replace (and X, (1 << size(X)-1) != 0) with X s< 0
1799 if (C2->isSignMask()) {
1800 Constant *Zero = Constant::getNullValue(X->getType());
1801 auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE;
1802 return new ICmpInst(NewPred, X, Zero);
1803 }
1804
1805 APInt NewC2 = *C2;
1806 KnownBits Know = computeKnownBits(And->getOperand(0), 0, And);
1807 // Set high zeros of C2 to allow matching negated power-of-2.
1808 NewC2 = *C2 | APInt::getHighBitsSet(C2->getBitWidth(),
1809 Know.countMinLeadingZeros());
1810
1811 // Restrict this fold only for single-use 'and' (PR10267).
1812 // ((%x & C) == 0) --> %x u< (-C) iff (-C) is power of two.
1813 if (NewC2.isNegatedPowerOf2()) {
1814 Constant *NegBOC = ConstantInt::get(And->getType(), -NewC2);
1815 auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT;
1816 return new ICmpInst(NewPred, X, NegBOC);
1817 }
1818 }
1819
1820 // If the LHS is an 'and' of a truncate and we can widen the and/compare to
1821 // the input width without changing the value produced, eliminate the cast:
1822 //
1823 // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1'
1824 //
1825 // We can do this transformation if the constants do not have their sign bits
1826 // set or if it is an equality comparison. Extending a relational comparison
1827 // when we're checking the sign bit would not work.
1828 Value *W;
1829 if (match(And->getOperand(0), m_OneUse(m_Trunc(m_Value(W)))) &&
1830 (Cmp.isEquality() || (!C1.isNegative() && !C2->isNegative()))) {
1831 // TODO: Is this a good transform for vectors? Wider types may reduce
1832 // throughput. Should this transform be limited (even for scalars) by using
1833 // shouldChangeType()?
1834 if (!Cmp.getType()->isVectorTy()) {
1835 Type *WideType = W->getType();
1836 unsigned WideScalarBits = WideType->getScalarSizeInBits();
1837 Constant *ZextC1 = ConstantInt::get(WideType, C1.zext(WideScalarBits));
1838 Constant *ZextC2 = ConstantInt::get(WideType, C2->zext(WideScalarBits));
1839 Value *NewAnd = Builder.CreateAnd(W, ZextC2, And->getName());
1840 return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1);
1841 }
1842 }
1843
1844 if (Instruction *I = foldICmpAndShift(Cmp, And, C1, *C2))
1845 return I;
1846
1847 // (icmp pred (and (or (lshr A, B), A), 1), 0) -->
1848 // (icmp pred (and A, (or (shl 1, B), 1), 0))
1849 //
1850 // iff pred isn't signed
1851 if (!Cmp.isSigned() && C1.isZero() && And->getOperand(0)->hasOneUse() &&
1852 match(And->getOperand(1), m_One())) {
1853 Constant *One = cast<Constant>(And->getOperand(1));
1854 Value *Or = And->getOperand(0);
1855 Value *A, *B, *LShr;
1856 if (match(Or, m_Or(m_Value(LShr), m_Value(A))) &&
1857 match(LShr, m_LShr(m_Specific(A), m_Value(B)))) {
1858 unsigned UsesRemoved = 0;
1859 if (And->hasOneUse())
1860 ++UsesRemoved;
1861 if (Or->hasOneUse())
1862 ++UsesRemoved;
1863 if (LShr->hasOneUse())
1864 ++UsesRemoved;
1865
1866 // Compute A & ((1 << B) | 1)
1867 unsigned RequireUsesRemoved = match(B, m_ImmConstant()) ? 1 : 3;
1868 if (UsesRemoved >= RequireUsesRemoved) {
1869 Value *NewOr =
1870 Builder.CreateOr(Builder.CreateShl(One, B, LShr->getName(),
1871 /*HasNUW=*/true),
1872 One, Or->getName());
1873 Value *NewAnd = Builder.CreateAnd(A, NewOr, And->getName());
1874 return new ICmpInst(Cmp.getPredicate(), NewAnd, Cmp.getOperand(1));
1875 }
1876 }
1877 }
1878
1879 // (icmp eq (and (bitcast X to int), ExponentMask), ExponentMask) -->
1880 // llvm.is.fpclass(X, fcInf|fcNan)
1881 // (icmp ne (and (bitcast X to int), ExponentMask), ExponentMask) -->
1882 // llvm.is.fpclass(X, ~(fcInf|fcNan))
1883 Value *V;
1884 if (!Cmp.getParent()->getParent()->hasFnAttribute(
1885 Attribute::NoImplicitFloat) &&
1886 Cmp.isEquality() &&
1888 Type *FPType = V->getType()->getScalarType();
1889 if (FPType->isIEEELikeFPTy() && C1 == *C2) {
1890 APInt ExponentMask =
1891 APFloat::getInf(FPType->getFltSemantics()).bitcastToAPInt();
1892 if (C1 == ExponentMask) {
1893 unsigned Mask = FPClassTest::fcNan | FPClassTest::fcInf;
1894 if (isICMP_NE)
1895 Mask = ~Mask & fcAllFlags;
1896 return replaceInstUsesWith(Cmp, Builder.createIsFPClass(V, Mask));
1897 }
1898 }
1899 }
1900
1901 return nullptr;
1902}
1903
1904/// Fold icmp (and X, Y), C.
1907 const APInt &C) {
1908 if (Instruction *I = foldICmpAndConstConst(Cmp, And, C))
1909 return I;
1910
1911 const ICmpInst::Predicate Pred = Cmp.getPredicate();
1912 bool TrueIfNeg;
1913 if (isSignBitCheck(Pred, C, TrueIfNeg)) {
1914 // ((X - 1) & ~X) < 0 --> X == 0
1915 // ((X - 1) & ~X) >= 0 --> X != 0
1916 Value *X;
1917 if (match(And->getOperand(0), m_Add(m_Value(X), m_AllOnes())) &&
1918 match(And->getOperand(1), m_Not(m_Specific(X)))) {
1919 auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1920 return new ICmpInst(NewPred, X, ConstantInt::getNullValue(X->getType()));
1921 }
1922 // (X & -X) < 0 --> X == MinSignedC
1923 // (X & -X) > -1 --> X != MinSignedC
1924 if (match(And, m_c_And(m_Neg(m_Value(X)), m_Deferred(X)))) {
1925 Constant *MinSignedC = ConstantInt::get(
1926 X->getType(),
1927 APInt::getSignedMinValue(X->getType()->getScalarSizeInBits()));
1928 auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1929 return new ICmpInst(NewPred, X, MinSignedC);
1930 }
1931 }
1932
1933 // TODO: These all require that Y is constant too, so refactor with the above.
1934
1935 // Try to optimize things like "A[i] & 42 == 0" to index computations.
1936 Value *X = And->getOperand(0);
1937 Value *Y = And->getOperand(1);
1938 if (auto *C2 = dyn_cast<ConstantInt>(Y))
1939 if (auto *LI = dyn_cast<LoadInst>(X))
1940 if (auto *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0)))
1941 if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)))
1942 if (Instruction *Res =
1943 foldCmpLoadFromIndexedGlobal(LI, GEP, GV, Cmp, C2))
1944 return Res;
1945
1946 if (!Cmp.isEquality())
1947 return nullptr;
1948
1949 // X & -C == -C -> X > u ~C
1950 // X & -C != -C -> X <= u ~C
1951 // iff C is a power of 2
1952 if (Cmp.getOperand(1) == Y && C.isNegatedPowerOf2()) {
1953 auto NewPred =
1955 return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1))));
1956 }
1957
1958 // If we are testing the intersection of 2 select-of-nonzero-constants with no
1959 // common bits set, it's the same as checking if exactly one select condition
1960 // is set:
1961 // ((A ? TC : FC) & (B ? TC : FC)) == 0 --> xor A, B
1962 // ((A ? TC : FC) & (B ? TC : FC)) != 0 --> not(xor A, B)
1963 // TODO: Generalize for non-constant values.
1964 // TODO: Handle signed/unsigned predicates.
1965 // TODO: Handle other bitwise logic connectors.
1966 // TODO: Extend to handle a non-zero compare constant.
1967 if (C.isZero() && (Pred == CmpInst::ICMP_EQ || And->hasOneUse())) {
1968 assert(Cmp.isEquality() && "Not expecting non-equality predicates");
1969 Value *A, *B;
1970 const APInt *TC, *FC;
1971 if (match(X, m_Select(m_Value(A), m_APInt(TC), m_APInt(FC))) &&
1972 match(Y,
1973 m_Select(m_Value(B), m_SpecificInt(*TC), m_SpecificInt(*FC))) &&
1974 !TC->isZero() && !FC->isZero() && !TC->intersects(*FC)) {
1975 Value *R = Builder.CreateXor(A, B);
1976 if (Pred == CmpInst::ICMP_NE)
1977 R = Builder.CreateNot(R);
1978 return replaceInstUsesWith(Cmp, R);
1979 }
1980 }
1981
1982 // ((zext i1 X) & Y) == 0 --> !((trunc Y) & X)
1983 // ((zext i1 X) & Y) != 0 --> ((trunc Y) & X)
1984 // ((zext i1 X) & Y) == 1 --> ((trunc Y) & X)
1985 // ((zext i1 X) & Y) != 1 --> !((trunc Y) & X)
1987 X->getType()->isIntOrIntVectorTy(1) && (C.isZero() || C.isOne())) {
1988 Value *TruncY = Builder.CreateTrunc(Y, X->getType());
1989 if (C.isZero() ^ (Pred == CmpInst::ICMP_NE)) {
1990 Value *And = Builder.CreateAnd(TruncY, X);
1992 }
1993 return BinaryOperator::CreateAnd(TruncY, X);
1994 }
1995
1996 // (icmp eq/ne (and (shl -1, X), Y), 0)
1997 // -> (icmp eq/ne (lshr Y, X), 0)
1998 // We could technically handle any C == 0 or (C < 0 && isOdd(C)) but it seems
1999 // highly unlikely the non-zero case will ever show up in code.
2000 if (C.isZero() &&
2002 m_Value(Y))))) {
2003 Value *LShr = Builder.CreateLShr(Y, X);
2004 return new ICmpInst(Pred, LShr, Constant::getNullValue(LShr->getType()));
2005 }
2006
2007 // (icmp eq/ne (and (add A, Addend), Msk), C)
2008 // -> (icmp eq/ne (and A, Msk), (and (sub C, Addend), Msk))
2009 {
2010 Value *A;
2011 const APInt *Addend, *Msk;
2012 if (match(And, m_And(m_OneUse(m_Add(m_Value(A), m_APInt(Addend))),
2013 m_APInt(Msk))) &&
2014 Msk->isMask() && C.ule(*Msk)) {
2015 APInt NewComperand = (C - *Addend) & *Msk;
2016 Value* MaskA = Builder.CreateAnd(A, ConstantInt::get(A->getType(), *Msk));
2017 return new ICmpInst(
2018 Pred, MaskA,
2019 Constant::getIntegerValue(MaskA->getType(), NewComperand));
2020 }
2021 }
2022
2023 return nullptr;
2024}
2025
2026/// Fold icmp eq/ne (or (xor/sub (X1, X2), xor/sub (X3, X4))), 0.
2028 InstCombiner::BuilderTy &Builder) {
2029 // Are we using xors or subs to bitwise check for a pair or pairs of
2030 // (in)equalities? Convert to a shorter form that has more potential to be
2031 // folded even further.
2032 // ((X1 ^/- X2) || (X3 ^/- X4)) == 0 --> (X1 == X2) && (X3 == X4)
2033 // ((X1 ^/- X2) || (X3 ^/- X4)) != 0 --> (X1 != X2) || (X3 != X4)
2034 // ((X1 ^/- X2) || (X3 ^/- X4) || (X5 ^/- X6)) == 0 -->
2035 // (X1 == X2) && (X3 == X4) && (X5 == X6)
2036 // ((X1 ^/- X2) || (X3 ^/- X4) || (X5 ^/- X6)) != 0 -->
2037 // (X1 != X2) || (X3 != X4) || (X5 != X6)
2039 SmallVector<Value *, 16> WorkList(1, Or);
2040
2041 while (!WorkList.empty()) {
2042 auto MatchOrOperatorArgument = [&](Value *OrOperatorArgument) {
2043 Value *Lhs, *Rhs;
2044
2045 if (match(OrOperatorArgument,
2046 m_OneUse(m_Xor(m_Value(Lhs), m_Value(Rhs))))) {
2047 CmpValues.emplace_back(Lhs, Rhs);
2048 return;
2049 }
2050
2051 if (match(OrOperatorArgument,
2052 m_OneUse(m_Sub(m_Value(Lhs), m_Value(Rhs))))) {
2053 CmpValues.emplace_back(Lhs, Rhs);
2054 return;
2055 }
2056
2057 WorkList.push_back(OrOperatorArgument);
2058 };
2059
2060 Value *CurrentValue = WorkList.pop_back_val();
2061 Value *OrOperatorLhs, *OrOperatorRhs;
2062
2063 if (!match(CurrentValue,
2064 m_Or(m_Value(OrOperatorLhs), m_Value(OrOperatorRhs)))) {
2065 return nullptr;
2066 }
2067
2068 MatchOrOperatorArgument(OrOperatorRhs);
2069 MatchOrOperatorArgument(OrOperatorLhs);
2070 }
2071
2072 ICmpInst::Predicate Pred = Cmp.getPredicate();
2073 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2074 Value *LhsCmp = Builder.CreateICmp(Pred, CmpValues.rbegin()->first,
2075 CmpValues.rbegin()->second);
2076
2077 for (auto It = CmpValues.rbegin() + 1; It != CmpValues.rend(); ++It) {
2078 Value *RhsCmp = Builder.CreateICmp(Pred, It->first, It->second);
2079 LhsCmp = Builder.CreateBinOp(BOpc, LhsCmp, RhsCmp);
2080 }
2081
2082 return LhsCmp;
2083}
2084
2085/// Fold icmp (or X, Y), C.
2088 const APInt &C) {
2089 ICmpInst::Predicate Pred = Cmp.getPredicate();
2090 if (C.isOne()) {
2091 // icmp slt signum(V) 1 --> icmp slt V, 1
2092 Value *V = nullptr;
2093 if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V))))
2094 return new ICmpInst(ICmpInst::ICMP_SLT, V,
2095 ConstantInt::get(V->getType(), 1));
2096 }
2097
2098 Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1);
2099
2100 // (icmp eq/ne (or disjoint x, C0), C1)
2101 // -> (icmp eq/ne x, C0^C1)
2102 if (Cmp.isEquality() && match(OrOp1, m_ImmConstant()) &&
2103 cast<PossiblyDisjointInst>(Or)->isDisjoint()) {
2104 Value *NewC =
2105 Builder.CreateXor(OrOp1, ConstantInt::get(OrOp1->getType(), C));
2106 return new ICmpInst(Pred, OrOp0, NewC);
2107 }
2108
2109 const APInt *MaskC;
2110 if (match(OrOp1, m_APInt(MaskC)) && Cmp.isEquality()) {
2111 if (*MaskC == C && (C + 1).isPowerOf2()) {
2112 // X | C == C --> X <=u C
2113 // X | C != C --> X >u C
2114 // iff C+1 is a power of 2 (C is a bitmask of the low bits)
2116 return new ICmpInst(Pred, OrOp0, OrOp1);
2117 }
2118
2119 // More general: canonicalize 'equality with set bits mask' to
2120 // 'equality with clear bits mask'.
2121 // (X | MaskC) == C --> (X & ~MaskC) == C ^ MaskC
2122 // (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC
2123 if (Or->hasOneUse()) {
2124 Value *And = Builder.CreateAnd(OrOp0, ~(*MaskC));
2125 Constant *NewC = ConstantInt::get(Or->getType(), C ^ (*MaskC));
2126 return new ICmpInst(Pred, And, NewC);
2127 }
2128 }
2129
2130 // (X | (X-1)) s< 0 --> X s< 1
2131 // (X | (X-1)) s> -1 --> X s> 0
2132 Value *X;
2133 bool TrueIfSigned;
2134 if (isSignBitCheck(Pred, C, TrueIfSigned) &&
2136 auto NewPred = TrueIfSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGT;
2137 Constant *NewC = ConstantInt::get(X->getType(), TrueIfSigned ? 1 : 0);
2138 return new ICmpInst(NewPred, X, NewC);
2139 }
2140
2141 const APInt *OrC;
2142 // icmp(X | OrC, C) --> icmp(X, 0)
2143 if (C.isNonNegative() && match(Or, m_Or(m_Value(X), m_APInt(OrC)))) {
2144 switch (Pred) {
2145 // X | OrC s< C --> X s< 0 iff OrC s>= C s>= 0
2146 case ICmpInst::ICMP_SLT:
2147 // X | OrC s>= C --> X s>= 0 iff OrC s>= C s>= 0
2148 case ICmpInst::ICMP_SGE:
2149 if (OrC->sge(C))
2150 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
2151 break;
2152 // X | OrC s<= C --> X s< 0 iff OrC s> C s>= 0
2153 case ICmpInst::ICMP_SLE:
2154 // X | OrC s> C --> X s>= 0 iff OrC s> C s>= 0
2155 case ICmpInst::ICMP_SGT:
2156 if (OrC->sgt(C))
2158 ConstantInt::getNullValue(X->getType()));
2159 break;
2160 default:
2161 break;
2162 }
2163 }
2164
2165 if (!Cmp.isEquality() || !C.isZero() || !Or->hasOneUse())
2166 return nullptr;
2167
2168 Value *P, *Q;
2170 // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0
2171 // -> and (icmp eq P, null), (icmp eq Q, null).
2172 Value *CmpP =
2173 Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType()));
2174 Value *CmpQ =
2176 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2177 return BinaryOperator::Create(BOpc, CmpP, CmpQ);
2178 }
2179
2180 if (Value *V = foldICmpOrXorSubChain(Cmp, Or, Builder))
2181 return replaceInstUsesWith(Cmp, V);
2182
2183 return nullptr;
2184}
2185
2186/// Fold icmp (mul X, Y), C.
2189 const APInt &C) {
2190 ICmpInst::Predicate Pred = Cmp.getPredicate();
2191 Type *MulTy = Mul->getType();
2192 Value *X = Mul->getOperand(0);
2193
2194 // If there's no overflow:
2195 // X * X == 0 --> X == 0
2196 // X * X != 0 --> X != 0
2197 if (Cmp.isEquality() && C.isZero() && X == Mul->getOperand(1) &&
2198 (Mul->hasNoUnsignedWrap() || Mul->hasNoSignedWrap()))
2199 return new ICmpInst(Pred, X, ConstantInt::getNullValue(MulTy));
2200
2201 const APInt *MulC;
2202 if (!match(Mul->getOperand(1), m_APInt(MulC)))
2203 return nullptr;
2204
2205 // If this is a test of the sign bit and the multiply is sign-preserving with
2206 // a constant operand, use the multiply LHS operand instead:
2207 // (X * +MulC) < 0 --> X < 0
2208 // (X * -MulC) < 0 --> X > 0
2209 if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) {
2210 if (MulC->isNegative())
2211 Pred = ICmpInst::getSwappedPredicate(Pred);
2212 return new ICmpInst(Pred, X, ConstantInt::getNullValue(MulTy));
2213 }
2214
2215 if (MulC->isZero())
2216 return nullptr;
2217
2218 // If the multiply does not wrap or the constant is odd, try to divide the
2219 // compare constant by the multiplication factor.
2220 if (Cmp.isEquality()) {
2221 // (mul nsw X, MulC) eq/ne C --> X eq/ne C /s MulC
2222 if (Mul->hasNoSignedWrap() && C.srem(*MulC).isZero()) {
2223 Constant *NewC = ConstantInt::get(MulTy, C.sdiv(*MulC));
2224 return new ICmpInst(Pred, X, NewC);
2225 }
2226
2227 // C % MulC == 0 is weaker than we could use if MulC is odd because it
2228 // correct to transform if MulC * N == C including overflow. I.e with i8
2229 // (icmp eq (mul X, 5), 101) -> (icmp eq X, 225) but since 101 % 5 != 0, we
2230 // miss that case.
2231 if (C.urem(*MulC).isZero()) {
2232 // (mul nuw X, MulC) eq/ne C --> X eq/ne C /u MulC
2233 // (mul X, OddC) eq/ne N * C --> X eq/ne N
2234 if ((*MulC & 1).isOne() || Mul->hasNoUnsignedWrap()) {
2235 Constant *NewC = ConstantInt::get(MulTy, C.udiv(*MulC));
2236 return new ICmpInst(Pred, X, NewC);
2237 }
2238 }
2239 }
2240
2241 // With a matching no-overflow guarantee, fold the constants:
2242 // (X * MulC) < C --> X < (C / MulC)
2243 // (X * MulC) > C --> X > (C / MulC)
2244 // TODO: Assert that Pred is not equal to SGE, SLE, UGE, ULE?
2245 Constant *NewC = nullptr;
2246 if (Mul->hasNoSignedWrap() && ICmpInst::isSigned(Pred)) {
2247 // MININT / -1 --> overflow.
2248 if (C.isMinSignedValue() && MulC->isAllOnes())
2249 return nullptr;
2250 if (MulC->isNegative())
2251 Pred = ICmpInst::getSwappedPredicate(Pred);
2252
2253 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) {
2254 NewC = ConstantInt::get(
2256 } else {
2257 assert((Pred == ICmpInst::ICMP_SLE || Pred == ICmpInst::ICMP_SGT) &&
2258 "Unexpected predicate");
2259 NewC = ConstantInt::get(
2261 }
2262 } else if (Mul->hasNoUnsignedWrap() && ICmpInst::isUnsigned(Pred)) {
2263 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) {
2264 NewC = ConstantInt::get(
2266 } else {
2267 assert((Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
2268 "Unexpected predicate");
2269 NewC = ConstantInt::get(
2271 }
2272 }
2273
2274 return NewC ? new ICmpInst(Pred, X, NewC) : nullptr;
2275}
2276
2277/// Fold icmp (shl nuw C2, Y), C.
2279 const APInt &C) {
2280 Value *Y;
2281 const APInt *C2;
2282 if (!match(Shl, m_NUWShl(m_APInt(C2), m_Value(Y))))
2283 return nullptr;
2284
2285 Type *ShiftType = Shl->getType();
2286 unsigned TypeBits = C.getBitWidth();
2287 ICmpInst::Predicate Pred = Cmp.getPredicate();
2288 if (Cmp.isUnsigned()) {
2289 if (C2->isZero() || C2->ugt(C))
2290 return nullptr;
2291 APInt Div, Rem;
2292 APInt::udivrem(C, *C2, Div, Rem);
2293 bool CIsPowerOf2 = Rem.isZero() && Div.isPowerOf2();
2294
2295 // (1 << Y) pred C -> Y pred Log2(C)
2296 if (!CIsPowerOf2) {
2297 // (1 << Y) < 30 -> Y <= 4
2298 // (1 << Y) <= 30 -> Y <= 4
2299 // (1 << Y) >= 30 -> Y > 4
2300 // (1 << Y) > 30 -> Y > 4
2301 if (Pred == ICmpInst::ICMP_ULT)
2302 Pred = ICmpInst::ICMP_ULE;
2303 else if (Pred == ICmpInst::ICMP_UGE)
2304 Pred = ICmpInst::ICMP_UGT;
2305 }
2306
2307 unsigned CLog2 = Div.logBase2();
2308 return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2));
2309 } else if (Cmp.isSigned() && C2->isOne()) {
2310 Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1);
2311 // (1 << Y) > 0 -> Y != 31
2312 // (1 << Y) > C -> Y != 31 if C is negative.
2313 if (Pred == ICmpInst::ICMP_SGT && C.sle(0))
2314 return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne);
2315
2316 // (1 << Y) < 0 -> Y == 31
2317 // (1 << Y) < 1 -> Y == 31
2318 // (1 << Y) < C -> Y == 31 if C is negative and not signed min.
2319 // Exclude signed min by subtracting 1 and lower the upper bound to 0.
2320 if (Pred == ICmpInst::ICMP_SLT && (C-1).sle(0))
2321 return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne);
2322 }
2323
2324 return nullptr;
2325}
2326
2327/// Fold icmp (shl X, Y), C.
2329 BinaryOperator *Shl,
2330 const APInt &C) {
2331 const APInt *ShiftVal;
2332 if (Cmp.isEquality() && match(Shl->getOperand(0), m_APInt(ShiftVal)))
2333 return foldICmpShlConstConst(Cmp, Shl->getOperand(1), C, *ShiftVal);
2334
2335 ICmpInst::Predicate Pred = Cmp.getPredicate();
2336 // (icmp pred (shl nuw&nsw X, Y), Csle0)
2337 // -> (icmp pred X, Csle0)
2338 //
2339 // The idea is the nuw/nsw essentially freeze the sign bit for the shift op
2340 // so X's must be what is used.
2341 if (C.sle(0) && Shl->hasNoUnsignedWrap() && Shl->hasNoSignedWrap())
2342 return new ICmpInst(Pred, Shl->getOperand(0), Cmp.getOperand(1));
2343
2344 // (icmp eq/ne (shl nuw|nsw X, Y), 0)
2345 // -> (icmp eq/ne X, 0)
2346 if (ICmpInst::isEquality(Pred) && C.isZero() &&
2347 (Shl->hasNoUnsignedWrap() || Shl->hasNoSignedWrap()))
2348 return new ICmpInst(Pred, Shl->getOperand(0), Cmp.getOperand(1));
2349
2350 // (icmp slt (shl nsw X, Y), 0/1)
2351 // -> (icmp slt X, 0/1)
2352 // (icmp sgt (shl nsw X, Y), 0/-1)
2353 // -> (icmp sgt X, 0/-1)
2354 //
2355 // NB: sge/sle with a constant will canonicalize to sgt/slt.
2356 if (Shl->hasNoSignedWrap() &&
2357 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT))
2358 if (C.isZero() || (Pred == ICmpInst::ICMP_SGT ? C.isAllOnes() : C.isOne()))
2359 return new ICmpInst(Pred, Shl->getOperand(0), Cmp.getOperand(1));
2360
2361 const APInt *ShiftAmt;
2362 if (!match(Shl->getOperand(1), m_APInt(ShiftAmt)))
2363 return foldICmpShlLHSC(Cmp, Shl, C);
2364
2365 // Check that the shift amount is in range. If not, don't perform undefined
2366 // shifts. When the shift is visited, it will be simplified.
2367 unsigned TypeBits = C.getBitWidth();
2368 if (ShiftAmt->uge(TypeBits))
2369 return nullptr;
2370
2371 Value *X = Shl->getOperand(0);
2372 Type *ShType = Shl->getType();
2373
2374 // NSW guarantees that we are only shifting out sign bits from the high bits,
2375 // so we can ASHR the compare constant without needing a mask and eliminate
2376 // the shift.
2377 if (Shl->hasNoSignedWrap()) {
2378 if (Pred == ICmpInst::ICMP_SGT) {
2379 // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt)
2380 APInt ShiftedC = C.ashr(*ShiftAmt);
2381 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2382 }
2383 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2384 C.ashr(*ShiftAmt).shl(*ShiftAmt) == C) {
2385 APInt ShiftedC = C.ashr(*ShiftAmt);
2386 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2387 }
2388 if (Pred == ICmpInst::ICMP_SLT) {
2389 // SLE is the same as above, but SLE is canonicalized to SLT, so convert:
2390 // (X << S) <=s C is equiv to X <=s (C >> S) for all C
2391 // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX
2392 // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN
2393 assert(!C.isMinSignedValue() && "Unexpected icmp slt");
2394 APInt ShiftedC = (C - 1).ashr(*ShiftAmt) + 1;
2395 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2396 }
2397 }
2398
2399 // NUW guarantees that we are only shifting out zero bits from the high bits,
2400 // so we can LSHR the compare constant without needing a mask and eliminate
2401 // the shift.
2402 if (Shl->hasNoUnsignedWrap()) {
2403 if (Pred == ICmpInst::ICMP_UGT) {
2404 // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt)
2405 APInt ShiftedC = C.lshr(*ShiftAmt);
2406 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2407 }
2408 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2409 C.lshr(*ShiftAmt).shl(*ShiftAmt) == C) {
2410 APInt ShiftedC = C.lshr(*ShiftAmt);
2411 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2412 }
2413 if (Pred == ICmpInst::ICMP_ULT) {
2414 // ULE is the same as above, but ULE is canonicalized to ULT, so convert:
2415 // (X << S) <=u C is equiv to X <=u (C >> S) for all C
2416 // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u
2417 // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0
2418 assert(C.ugt(0) && "ult 0 should have been eliminated");
2419 APInt ShiftedC = (C - 1).lshr(*ShiftAmt) + 1;
2420 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2421 }
2422 }
2423
2424 if (Cmp.isEquality() && Shl->hasOneUse()) {
2425 // Strength-reduce the shift into an 'and'.
2426 Constant *Mask = ConstantInt::get(
2427 ShType,
2428 APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt->getZExtValue()));
2429 Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask");
2430 Constant *LShrC = ConstantInt::get(ShType, C.lshr(*ShiftAmt));
2431 return new ICmpInst(Pred, And, LShrC);
2432 }
2433
2434 // Otherwise, if this is a comparison of the sign bit, simplify to and/test.
2435 bool TrueIfSigned = false;
2436 if (Shl->hasOneUse() && isSignBitCheck(Pred, C, TrueIfSigned)) {
2437 // (X << 31) <s 0 --> (X & 1) != 0
2438 Constant *Mask = ConstantInt::get(
2439 ShType,
2440 APInt::getOneBitSet(TypeBits, TypeBits - ShiftAmt->getZExtValue() - 1));
2441 Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask");
2442 return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ,
2443 And, Constant::getNullValue(ShType));
2444 }
2445
2446 // Simplify 'shl' inequality test into 'and' equality test.
2447 if (Cmp.isUnsigned() && Shl->hasOneUse()) {
2448 // (X l<< C2) u<=/u> C1 iff C1+1 is power of two -> X & (~C1 l>> C2) ==/!= 0
2449 if ((C + 1).isPowerOf2() &&
2450 (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)) {
2451 Value *And = Builder.CreateAnd(X, (~C).lshr(ShiftAmt->getZExtValue()));
2452 return new ICmpInst(Pred == ICmpInst::ICMP_ULE ? ICmpInst::ICMP_EQ
2454 And, Constant::getNullValue(ShType));
2455 }
2456 // (X l<< C2) u</u>= C1 iff C1 is power of two -> X & (-C1 l>> C2) ==/!= 0
2457 if (C.isPowerOf2() &&
2458 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) {
2459 Value *And =
2460 Builder.CreateAnd(X, (~(C - 1)).lshr(ShiftAmt->getZExtValue()));
2461 return new ICmpInst(Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_EQ
2463 And, Constant::getNullValue(ShType));
2464 }
2465 }
2466
2467 // Transform (icmp pred iM (shl iM %v, N), C)
2468 // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N))
2469 // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N.
2470 // This enables us to get rid of the shift in favor of a trunc that may be
2471 // free on the target. It has the additional benefit of comparing to a
2472 // smaller constant that may be more target-friendly.
2473 unsigned Amt = ShiftAmt->getLimitedValue(TypeBits - 1);
2474 if (Shl->hasOneUse() && Amt != 0 &&
2475 shouldChangeType(ShType->getScalarSizeInBits(), TypeBits - Amt)) {
2476 ICmpInst::Predicate CmpPred = Pred;
2477 APInt RHSC = C;
2478
2479 if (RHSC.countr_zero() < Amt && ICmpInst::isStrictPredicate(CmpPred)) {
2480 // Try the flipped strictness predicate.
2481 // e.g.:
2482 // icmp ult i64 (shl X, 32), 8589934593 ->
2483 // icmp ule i64 (shl X, 32), 8589934592 ->
2484 // icmp ule i32 (trunc X, i32), 2 ->
2485 // icmp ult i32 (trunc X, i32), 3
2486 if (auto FlippedStrictness =
2488 Pred, ConstantInt::get(ShType->getContext(), C))) {
2489 CmpPred = FlippedStrictness->first;
2490 RHSC = cast<ConstantInt>(FlippedStrictness->second)->getValue();
2491 }
2492 }
2493
2494 if (RHSC.countr_zero() >= Amt) {
2495 Type *TruncTy = ShType->getWithNewBitWidth(TypeBits - Amt);
2496 Constant *NewC =
2497 ConstantInt::get(TruncTy, RHSC.ashr(*ShiftAmt).trunc(TypeBits - Amt));
2498 return new ICmpInst(CmpPred,
2499 Builder.CreateTrunc(X, TruncTy, "", /*IsNUW=*/false,
2500 Shl->hasNoSignedWrap()),
2501 NewC);
2502 }
2503 }
2504
2505 return nullptr;
2506}
2507
2508/// Fold icmp ({al}shr X, Y), C.
2510 BinaryOperator *Shr,
2511 const APInt &C) {
2512 // An exact shr only shifts out zero bits, so:
2513 // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0
2514 Value *X = Shr->getOperand(0);
2515 CmpInst::Predicate Pred = Cmp.getPredicate();
2516 if (Cmp.isEquality() && Shr->isExact() && C.isZero())
2517 return new ICmpInst(Pred, X, Cmp.getOperand(1));
2518
2519 bool IsAShr = Shr->getOpcode() == Instruction::AShr;
2520 const APInt *ShiftValC;
2521 if (match(X, m_APInt(ShiftValC))) {
2522 if (Cmp.isEquality())
2523 return foldICmpShrConstConst(Cmp, Shr->getOperand(1), C, *ShiftValC);
2524
2525 // (ShiftValC >> Y) >s -1 --> Y != 0 with ShiftValC < 0
2526 // (ShiftValC >> Y) <s 0 --> Y == 0 with ShiftValC < 0
2527 bool TrueIfSigned;
2528 if (!IsAShr && ShiftValC->isNegative() &&
2529 isSignBitCheck(Pred, C, TrueIfSigned))
2530 return new ICmpInst(TrueIfSigned ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE,
2531 Shr->getOperand(1),
2532 ConstantInt::getNullValue(X->getType()));
2533
2534 // If the shifted constant is a power-of-2, test the shift amount directly:
2535 // (ShiftValC >> Y) >u C --> X <u (LZ(C) - LZ(ShiftValC))
2536 // (ShiftValC >> Y) <u C --> X >=u (LZ(C-1) - LZ(ShiftValC))
2537 if (!IsAShr && ShiftValC->isPowerOf2() &&
2538 (Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_ULT)) {
2539 bool IsUGT = Pred == CmpInst::ICMP_UGT;
2540 assert(ShiftValC->uge(C) && "Expected simplify of compare");
2541 assert((IsUGT || !C.isZero()) && "Expected X u< 0 to simplify");
2542
2543 unsigned CmpLZ = IsUGT ? C.countl_zero() : (C - 1).countl_zero();
2544 unsigned ShiftLZ = ShiftValC->countl_zero();
2545 Constant *NewC = ConstantInt::get(Shr->getType(), CmpLZ - ShiftLZ);
2546 auto NewPred = IsUGT ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE;
2547 return new ICmpInst(NewPred, Shr->getOperand(1), NewC);
2548 }
2549 }
2550
2551 const APInt *ShiftAmtC;
2552 if (!match(Shr->getOperand(1), m_APInt(ShiftAmtC)))
2553 return nullptr;
2554
2555 // Check that the shift amount is in range. If not, don't perform undefined
2556 // shifts. When the shift is visited it will be simplified.
2557 unsigned TypeBits = C.getBitWidth();
2558 unsigned ShAmtVal = ShiftAmtC->getLimitedValue(TypeBits);
2559 if (ShAmtVal >= TypeBits || ShAmtVal == 0)
2560 return nullptr;
2561
2562 bool IsExact = Shr->isExact();
2563 Type *ShrTy = Shr->getType();
2564 // TODO: If we could guarantee that InstSimplify would handle all of the
2565 // constant-value-based preconditions in the folds below, then we could assert
2566 // those conditions rather than checking them. This is difficult because of
2567 // undef/poison (PR34838).
2568 if (IsAShr && Shr->hasOneUse()) {
2569 if (IsExact && (Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) &&
2570 (C - 1).isPowerOf2() && C.countLeadingZeros() > ShAmtVal) {
2571 // When C - 1 is a power of two and the transform can be legally
2572 // performed, prefer this form so the produced constant is close to a
2573 // power of two.
2574 // icmp slt/ult (ashr exact X, ShAmtC), C
2575 // --> icmp slt/ult X, (C - 1) << ShAmtC) + 1
2576 APInt ShiftedC = (C - 1).shl(ShAmtVal) + 1;
2577 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2578 }
2579 if (IsExact || Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) {
2580 // When ShAmtC can be shifted losslessly:
2581 // icmp PRED (ashr exact X, ShAmtC), C --> icmp PRED X, (C << ShAmtC)
2582 // icmp slt/ult (ashr X, ShAmtC), C --> icmp slt/ult X, (C << ShAmtC)
2583 APInt ShiftedC = C.shl(ShAmtVal);
2584 if (ShiftedC.ashr(ShAmtVal) == C)
2585 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2586 }
2587 if (Pred == CmpInst::ICMP_SGT) {
2588 // icmp sgt (ashr X, ShAmtC), C --> icmp sgt X, ((C + 1) << ShAmtC) - 1
2589 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2590 if (!C.isMaxSignedValue() && !(C + 1).shl(ShAmtVal).isMinSignedValue() &&
2591 (ShiftedC + 1).ashr(ShAmtVal) == (C + 1))
2592 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2593 }
2594 if (Pred == CmpInst::ICMP_UGT) {
2595 // icmp ugt (ashr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1
2596 // 'C + 1 << ShAmtC' can overflow as a signed number, so the 2nd
2597 // clause accounts for that pattern.
2598 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2599 if ((ShiftedC + 1).ashr(ShAmtVal) == (C + 1) ||
2600 (C + 1).shl(ShAmtVal).isMinSignedValue())
2601 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2602 }
2603
2604 // If the compare constant has significant bits above the lowest sign-bit,
2605 // then convert an unsigned cmp to a test of the sign-bit:
2606 // (ashr X, ShiftC) u> C --> X s< 0
2607 // (ashr X, ShiftC) u< C --> X s> -1
2608 if (C.getBitWidth() > 2 && C.getNumSignBits() <= ShAmtVal) {
2609 if (Pred == CmpInst::ICMP_UGT) {
2610 return new ICmpInst(CmpInst::ICMP_SLT, X,
2612 }
2613 if (Pred == CmpInst::ICMP_ULT) {
2614 return new ICmpInst(CmpInst::ICMP_SGT, X,
2616 }
2617 }
2618 } else if (!IsAShr) {
2619 if (Pred == CmpInst::ICMP_ULT || (Pred == CmpInst::ICMP_UGT && IsExact)) {
2620 // icmp ult (lshr X, ShAmtC), C --> icmp ult X, (C << ShAmtC)
2621 // icmp ugt (lshr exact X, ShAmtC), C --> icmp ugt X, (C << ShAmtC)
2622 APInt ShiftedC = C.shl(ShAmtVal);
2623 if (ShiftedC.lshr(ShAmtVal) == C)
2624 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2625 }
2626 if (Pred == CmpInst::ICMP_UGT) {
2627 // icmp ugt (lshr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1
2628 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2629 if ((ShiftedC + 1).lshr(ShAmtVal) == (C + 1))
2630 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2631 }
2632 }
2633
2634 if (!Cmp.isEquality())
2635 return nullptr;
2636
2637 // Handle equality comparisons of shift-by-constant.
2638
2639 // If the comparison constant changes with the shift, the comparison cannot
2640 // succeed (bits of the comparison constant cannot match the shifted value).
2641 // This should be known by InstSimplify and already be folded to true/false.
2642 assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) ||
2643 (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) &&
2644 "Expected icmp+shr simplify did not occur.");
2645
2646 // If the bits shifted out are known zero, compare the unshifted value:
2647 // (X & 4) >> 1 == 2 --> (X & 4) == 4.
2648 if (Shr->isExact())
2649 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, C << ShAmtVal));
2650
2651 if (C.isZero()) {
2652 // == 0 is u< 1.
2653 if (Pred == CmpInst::ICMP_EQ)
2654 return new ICmpInst(CmpInst::ICMP_ULT, X,
2655 ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal)));
2656 else
2657 return new ICmpInst(CmpInst::ICMP_UGT, X,
2658 ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal) - 1));
2659 }
2660
2661 if (Shr->hasOneUse()) {
2662 // Canonicalize the shift into an 'and':
2663 // icmp eq/ne (shr X, ShAmt), C --> icmp eq/ne (and X, HiMask), (C << ShAmt)
2664 APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal));
2665 Constant *Mask = ConstantInt::get(ShrTy, Val);
2666 Value *And = Builder.CreateAnd(X, Mask, Shr->getName() + ".mask");
2667 return new ICmpInst(Pred, And, ConstantInt::get(ShrTy, C << ShAmtVal));
2668 }
2669
2670 return nullptr;
2671}
2672
2674 BinaryOperator *SRem,
2675 const APInt &C) {
2676 // Match an 'is positive' or 'is negative' comparison of remainder by a
2677 // constant power-of-2 value:
2678 // (X % pow2C) sgt/slt 0
2679 const ICmpInst::Predicate Pred = Cmp.getPredicate();
2680 if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_SLT &&
2681 Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
2682 return nullptr;
2683
2684 // TODO: The one-use check is standard because we do not typically want to
2685 // create longer instruction sequences, but this might be a special-case
2686 // because srem is not good for analysis or codegen.
2687 if (!SRem->hasOneUse())
2688 return nullptr;
2689
2690 const APInt *DivisorC;
2691 if (!match(SRem->getOperand(1), m_Power2(DivisorC)))
2692 return nullptr;
2693
2694 // For cmp_sgt/cmp_slt only zero valued C is handled.
2695 // For cmp_eq/cmp_ne only positive valued C is handled.
2696 if (((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT) &&
2697 !C.isZero()) ||
2698 ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2699 !C.isStrictlyPositive()))
2700 return nullptr;
2701
2702 // Mask off the sign bit and the modulo bits (low-bits).
2703 Type *Ty = SRem->getType();
2705 Constant *MaskC = ConstantInt::get(Ty, SignMask | (*DivisorC - 1));
2706 Value *And = Builder.CreateAnd(SRem->getOperand(0), MaskC);
2707
2708 if (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE)
2709 return new ICmpInst(Pred, And, ConstantInt::get(Ty, C));
2710
2711 // For 'is positive?' check that the sign-bit is clear and at least 1 masked
2712 // bit is set. Example:
2713 // (i8 X % 32) s> 0 --> (X & 159) s> 0
2714 if (Pred == ICmpInst::ICMP_SGT)
2716
2717 // For 'is negative?' check that the sign-bit is set and at least 1 masked
2718 // bit is set. Example:
2719 // (i16 X % 4) s< 0 --> (X & 32771) u> 32768
2720 return new ICmpInst(ICmpInst::ICMP_UGT, And, ConstantInt::get(Ty, SignMask));
2721}
2722
2723/// Fold icmp (udiv X, Y), C.
2725 BinaryOperator *UDiv,
2726 const APInt &C) {
2727 ICmpInst::Predicate Pred = Cmp.getPredicate();
2728 Value *X = UDiv->getOperand(0);
2729 Value *Y = UDiv->getOperand(1);
2730 Type *Ty = UDiv->getType();
2731
2732 const APInt *C2;
2733 if (!match(X, m_APInt(C2)))
2734 return nullptr;
2735
2736 assert(*C2 != 0 && "udiv 0, X should have been simplified already.");
2737
2738 // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1))
2739 if (Pred == ICmpInst::ICMP_UGT) {
2740 assert(!C.isMaxValue() &&
2741 "icmp ugt X, UINT_MAX should have been simplified already.");
2742 return new ICmpInst(ICmpInst::ICMP_ULE, Y,
2743 ConstantInt::get(Ty, C2->udiv(C + 1)));
2744 }
2745
2746 // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C)
2747 if (Pred == ICmpInst::ICMP_ULT) {
2748 assert(C != 0 && "icmp ult X, 0 should have been simplified already.");
2749 return new ICmpInst(ICmpInst::ICMP_UGT, Y,
2750 ConstantInt::get(Ty, C2->udiv(C)));
2751 }
2752
2753 return nullptr;
2754}
2755
2756/// Fold icmp ({su}div X, Y), C.
2758 BinaryOperator *Div,
2759 const APInt &C) {
2760 ICmpInst::Predicate Pred = Cmp.getPredicate();
2761 Value *X = Div->getOperand(0);
2762 Value *Y = Div->getOperand(1);
2763 Type *Ty = Div->getType();
2764 bool DivIsSigned = Div->getOpcode() == Instruction::SDiv;
2765
2766 // If unsigned division and the compare constant is bigger than
2767 // UMAX/2 (negative), there's only one pair of values that satisfies an
2768 // equality check, so eliminate the division:
2769 // (X u/ Y) == C --> (X == C) && (Y == 1)
2770 // (X u/ Y) != C --> (X != C) || (Y != 1)
2771 // Similarly, if signed division and the compare constant is exactly SMIN:
2772 // (X s/ Y) == SMIN --> (X == SMIN) && (Y == 1)
2773 // (X s/ Y) != SMIN --> (X != SMIN) || (Y != 1)
2774 if (Cmp.isEquality() && Div->hasOneUse() && C.isSignBitSet() &&
2775 (!DivIsSigned || C.isMinSignedValue())) {
2776 Value *XBig = Builder.CreateICmp(Pred, X, ConstantInt::get(Ty, C));
2777 Value *YOne = Builder.CreateICmp(Pred, Y, ConstantInt::get(Ty, 1));
2778 auto Logic = Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2779 return BinaryOperator::Create(Logic, XBig, YOne);
2780 }
2781
2782 // Fold: icmp pred ([us]div X, C2), C -> range test
2783 // Fold this div into the comparison, producing a range check.
2784 // Determine, based on the divide type, what the range is being
2785 // checked. If there is an overflow on the low or high side, remember
2786 // it, otherwise compute the range [low, hi) bounding the new value.
2787 // See: InsertRangeTest above for the kinds of replacements possible.
2788 const APInt *C2;
2789 if (!match(Y, m_APInt(C2)))
2790 return nullptr;
2791
2792 // FIXME: If the operand types don't match the type of the divide
2793 // then don't attempt this transform. The code below doesn't have the
2794 // logic to deal with a signed divide and an unsigned compare (and
2795 // vice versa). This is because (x /s C2) <s C produces different
2796 // results than (x /s C2) <u C or (x /u C2) <s C or even
2797 // (x /u C2) <u C. Simply casting the operands and result won't
2798 // work. :( The if statement below tests that condition and bails
2799 // if it finds it.
2800 if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned())
2801 return nullptr;
2802
2803 // The ProdOV computation fails on divide by 0 and divide by -1. Cases with
2804 // INT_MIN will also fail if the divisor is 1. Although folds of all these
2805 // division-by-constant cases should be present, we can not assert that they
2806 // have happened before we reach this icmp instruction.
2807 if (C2->isZero() || C2->isOne() || (DivIsSigned && C2->isAllOnes()))
2808 return nullptr;
2809
2810 // Compute Prod = C * C2. We are essentially solving an equation of
2811 // form X / C2 = C. We solve for X by multiplying C2 and C.
2812 // By solving for X, we can turn this into a range check instead of computing
2813 // a divide.
2814 APInt Prod = C * *C2;
2815
2816 // Determine if the product overflows by seeing if the product is not equal to
2817 // the divide. Make sure we do the same kind of divide as in the LHS
2818 // instruction that we're folding.
2819 bool ProdOV = (DivIsSigned ? Prod.sdiv(*C2) : Prod.udiv(*C2)) != C;
2820
2821 // If the division is known to be exact, then there is no remainder from the
2822 // divide, so the covered range size is unit, otherwise it is the divisor.
2823 APInt RangeSize = Div->isExact() ? APInt(C2->getBitWidth(), 1) : *C2;
2824
2825 // Figure out the interval that is being checked. For example, a comparison
2826 // like "X /u 5 == 0" is really checking that X is in the interval [0, 5).
2827 // Compute this interval based on the constants involved and the signedness of
2828 // the compare/divide. This computes a half-open interval, keeping track of
2829 // whether either value in the interval overflows. After analysis each
2830 // overflow variable is set to 0 if it's corresponding bound variable is valid
2831 // -1 if overflowed off the bottom end, or +1 if overflowed off the top end.
2832 int LoOverflow = 0, HiOverflow = 0;
2833 APInt LoBound, HiBound;
2834
2835 if (!DivIsSigned) { // udiv
2836 // e.g. X/5 op 3 --> [15, 20)
2837 LoBound = Prod;
2838 HiOverflow = LoOverflow = ProdOV;
2839 if (!HiOverflow) {
2840 // If this is not an exact divide, then many values in the range collapse
2841 // to the same result value.
2842 HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false);
2843 }
2844 } else if (C2->isStrictlyPositive()) { // Divisor is > 0.
2845 if (C.isZero()) { // (X / pos) op 0
2846 // Can't overflow. e.g. X/2 op 0 --> [-1, 2)
2847 LoBound = -(RangeSize - 1);
2848 HiBound = RangeSize;
2849 } else if (C.isStrictlyPositive()) { // (X / pos) op pos
2850 LoBound = Prod; // e.g. X/5 op 3 --> [15, 20)
2851 HiOverflow = LoOverflow = ProdOV;
2852 if (!HiOverflow)
2853 HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true);
2854 } else { // (X / pos) op neg
2855 // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14)
2856 HiBound = Prod + 1;
2857 LoOverflow = HiOverflow = ProdOV ? -1 : 0;
2858 if (!LoOverflow) {
2859 APInt DivNeg = -RangeSize;
2860 LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0;
2861 }
2862 }
2863 } else if (C2->isNegative()) { // Divisor is < 0.
2864 if (Div->isExact())
2865 RangeSize.negate();
2866 if (C.isZero()) { // (X / neg) op 0
2867 // e.g. X/-5 op 0 --> [-4, 5)
2868 LoBound = RangeSize + 1;
2869 HiBound = -RangeSize;
2870 if (HiBound == *C2) { // -INTMIN = INTMIN
2871 HiOverflow = 1; // [INTMIN+1, overflow)
2872 HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN
2873 }
2874 } else if (C.isStrictlyPositive()) { // (X / neg) op pos
2875 // e.g. X/-5 op 3 --> [-19, -14)
2876 HiBound = Prod + 1;
2877 HiOverflow = LoOverflow = ProdOV ? -1 : 0;
2878 if (!LoOverflow)
2879 LoOverflow =
2880 addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1 : 0;
2881 } else { // (X / neg) op neg
2882 LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20)
2883 LoOverflow = HiOverflow = ProdOV;
2884 if (!HiOverflow)
2885 HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true);
2886 }
2887
2888 // Dividing by a negative swaps the condition. LT <-> GT
2889 Pred = ICmpInst::getSwappedPredicate(Pred);
2890 }
2891
2892 switch (Pred) {
2893 default:
2894 llvm_unreachable("Unhandled icmp predicate!");
2895 case ICmpInst::ICMP_EQ:
2896 if (LoOverflow && HiOverflow)
2897 return replaceInstUsesWith(Cmp, Builder.getFalse());
2898 if (HiOverflow)
2899 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
2900 X, ConstantInt::get(Ty, LoBound));
2901 if (LoOverflow)
2902 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
2903 X, ConstantInt::get(Ty, HiBound));
2904 return replaceInstUsesWith(
2905 Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, true));
2906 case ICmpInst::ICMP_NE:
2907 if (LoOverflow && HiOverflow)
2908 return replaceInstUsesWith(Cmp, Builder.getTrue());
2909 if (HiOverflow)
2910 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
2911 X, ConstantInt::get(Ty, LoBound));
2912 if (LoOverflow)
2913 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
2914 X, ConstantInt::get(Ty, HiBound));
2915 return replaceInstUsesWith(
2916 Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, false));
2917 case ICmpInst::ICMP_ULT:
2918 case ICmpInst::ICMP_SLT:
2919 if (LoOverflow == +1) // Low bound is greater than input range.
2920 return replaceInstUsesWith(Cmp, Builder.getTrue());
2921 if (LoOverflow == -1) // Low bound is less than input range.
2922 return replaceInstUsesWith(Cmp, Builder.getFalse());
2923 return new ICmpInst(Pred, X, ConstantInt::get(Ty, LoBound));
2924 case ICmpInst::ICMP_UGT:
2925 case ICmpInst::ICMP_SGT:
2926 if (HiOverflow == +1) // High bound greater than input range.
2927 return replaceInstUsesWith(Cmp, Builder.getFalse());
2928 if (HiOverflow == -1) // High bound less than input range.
2929 return replaceInstUsesWith(Cmp, Builder.getTrue());
2930 if (Pred == ICmpInst::ICMP_UGT)
2931 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, HiBound));
2932 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, HiBound));
2933 }
2934
2935 return nullptr;
2936}
2937
2938/// Fold icmp (sub X, Y), C.
2940 BinaryOperator *Sub,
2941 const APInt &C) {
2942 Value *X = Sub->getOperand(0), *Y = Sub->getOperand(1);
2943 ICmpInst::Predicate Pred = Cmp.getPredicate();
2944 Type *Ty = Sub->getType();
2945
2946 // (SubC - Y) == C) --> Y == (SubC - C)
2947 // (SubC - Y) != C) --> Y != (SubC - C)
2948 Constant *SubC;
2949 if (Cmp.isEquality() && match(X, m_ImmConstant(SubC))) {
2950 return new ICmpInst(Pred, Y,
2951 ConstantExpr::getSub(SubC, ConstantInt::get(Ty, C)));
2952 }
2953
2954 // (icmp P (sub nuw|nsw C2, Y), C) -> (icmp swap(P) Y, C2-C)
2955 const APInt *C2;
2956 APInt SubResult;
2957 ICmpInst::Predicate SwappedPred = Cmp.getSwappedPredicate();
2958 bool HasNSW = Sub->hasNoSignedWrap();
2959 bool HasNUW = Sub->hasNoUnsignedWrap();
2960 if (match(X, m_APInt(C2)) &&
2961 ((Cmp.isUnsigned() && HasNUW) || (Cmp.isSigned() && HasNSW)) &&
2962 !subWithOverflow(SubResult, *C2, C, Cmp.isSigned()))
2963 return new ICmpInst(SwappedPred, Y, ConstantInt::get(Ty, SubResult));
2964
2965 // X - Y == 0 --> X == Y.
2966 // X - Y != 0 --> X != Y.
2967 // TODO: We allow this with multiple uses as long as the other uses are not
2968 // in phis. The phi use check is guarding against a codegen regression
2969 // for a loop test. If the backend could undo this (and possibly
2970 // subsequent transforms), we would not need this hack.
2971 if (Cmp.isEquality() && C.isZero() &&
2972 none_of((Sub->users()), [](const User *U) { return isa<PHINode>(U); }))
2973 return new ICmpInst(Pred, X, Y);
2974
2975 // The following transforms are only worth it if the only user of the subtract
2976 // is the icmp.
2977 // TODO: This is an artificial restriction for all of the transforms below
2978 // that only need a single replacement icmp. Can these use the phi test
2979 // like the transform above here?
2980 if (!Sub->hasOneUse())
2981 return nullptr;
2982
2983 if (Sub->hasNoSignedWrap()) {
2984 // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y)
2985 if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
2986 return new ICmpInst(ICmpInst::ICMP_SGE, X, Y);
2987
2988 // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y)
2989 if (Pred == ICmpInst::ICMP_SGT && C.isZero())
2990 return new ICmpInst(ICmpInst::ICMP_SGT, X, Y);
2991
2992 // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y)
2993 if (Pred == ICmpInst::ICMP_SLT && C.isZero())
2994 return new ICmpInst(ICmpInst::ICMP_SLT, X, Y);
2995
2996 // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y)
2997 if (Pred == ICmpInst::ICMP_SLT && C.isOne())
2998 return new ICmpInst(ICmpInst::ICMP_SLE, X, Y);
2999 }
3000
3001 if (!match(X, m_APInt(C2)))
3002 return nullptr;
3003
3004 // C2 - Y <u C -> (Y | (C - 1)) == C2
3005 // iff (C2 & (C - 1)) == C - 1 and C is a power of 2
3006 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() &&
3007 (*C2 & (C - 1)) == (C - 1))
3008 return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(Y, C - 1), X);
3009
3010 // C2 - Y >u C -> (Y | C) != C2
3011 // iff C2 & C == C and C + 1 is a power of 2
3012 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == C)
3013 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(Y, C), X);
3014
3015 // We have handled special cases that reduce.
3016 // Canonicalize any remaining sub to add as:
3017 // (C2 - Y) > C --> (Y + ~C2) < ~C
3018 Value *Add = Builder.CreateAdd(Y, ConstantInt::get(Ty, ~(*C2)), "notsub",
3019 HasNUW, HasNSW);
3020 return new ICmpInst(SwappedPred, Add, ConstantInt::get(Ty, ~C));
3021}
3022
3023static Value *createLogicFromTable(const std::bitset<4> &Table, Value *Op0,
3024 Value *Op1, IRBuilderBase &Builder,
3025 bool HasOneUse) {
3026 auto FoldConstant = [&](bool Val) {
3027 Constant *Res = Val ? Builder.getTrue() : Builder.getFalse();
3028 if (Op0->getType()->isVectorTy())
3030 cast<VectorType>(Op0->getType())->getElementCount(), Res);
3031 return Res;
3032 };
3033
3034 switch (Table.to_ulong()) {
3035 case 0: // 0 0 0 0
3036 return FoldConstant(false);
3037 case 1: // 0 0 0 1
3038 return HasOneUse ? Builder.CreateNot(Builder.CreateOr(Op0, Op1)) : nullptr;
3039 case 2: // 0 0 1 0
3040 return HasOneUse ? Builder.CreateAnd(Builder.CreateNot(Op0), Op1) : nullptr;
3041 case 3: // 0 0 1 1
3042 return Builder.CreateNot(Op0);
3043 case 4: // 0 1 0 0
3044 return HasOneUse ? Builder.CreateAnd(Op0, Builder.CreateNot(Op1)) : nullptr;
3045 case 5: // 0 1 0 1
3046 return Builder.CreateNot(Op1);
3047 case 6: // 0 1 1 0
3048 return Builder.CreateXor(Op0, Op1);
3049 case 7: // 0 1 1 1
3050 return HasOneUse ? Builder.CreateNot(Builder.CreateAnd(Op0, Op1)) : nullptr;
3051 case 8: // 1 0 0 0
3052 return Builder.CreateAnd(Op0, Op1);
3053 case 9: // 1 0 0 1
3054 return HasOneUse ? Builder.CreateNot(Builder.CreateXor(Op0, Op1)) : nullptr;
3055 case 10: // 1 0 1 0
3056 return Op1;
3057 case 11: // 1 0 1 1
3058 return HasOneUse ? Builder.CreateOr(Builder.CreateNot(Op0), Op1) : nullptr;
3059 case 12: // 1 1 0 0
3060 return Op0;
3061 case 13: // 1 1 0 1
3062 return HasOneUse ? Builder.CreateOr(Op0, Builder.CreateNot(Op1)) : nullptr;
3063 case 14: // 1 1 1 0
3064 return Builder.CreateOr(Op0, Op1);
3065 case 15: // 1 1 1 1
3066 return FoldConstant(true);
3067 default:
3068 llvm_unreachable("Invalid Operation");
3069 }
3070 return nullptr;
3071}
3072
3073/// Fold icmp (add X, Y), C.
3076 const APInt &C) {
3077 Value *Y = Add->getOperand(1);
3078 Value *X = Add->getOperand(0);
3079
3080 Value *Op0, *Op1;
3081 Instruction *Ext0, *Ext1;
3082 const CmpInst::Predicate Pred = Cmp.getPredicate();
3083 if (match(Add,
3086 m_ZExtOrSExt(m_Value(Op1))))) &&
3087 Op0->getType()->isIntOrIntVectorTy(1) &&
3088 Op1->getType()->isIntOrIntVectorTy(1)) {
3089 unsigned BW = C.getBitWidth();
3090 std::bitset<4> Table;
3091 auto ComputeTable = [&](bool Op0Val, bool Op1Val) {
3092 int Res = 0;
3093 if (Op0Val)
3094 Res += isa<ZExtInst>(Ext0) ? 1 : -1;
3095 if (Op1Val)
3096 Res += isa<ZExtInst>(Ext1) ? 1 : -1;
3097 return ICmpInst::compare(APInt(BW, Res, true), C, Pred);
3098 };
3099
3100 Table[0] = ComputeTable(false, false);
3101 Table[1] = ComputeTable(false, true);
3102 Table[2] = ComputeTable(true, false);
3103 Table[3] = ComputeTable(true, true);
3104 if (auto *Cond =
3105 createLogicFromTable(Table, Op0, Op1, Builder, Add->hasOneUse()))
3106 return replaceInstUsesWith(Cmp, Cond);
3107 }
3108 const APInt *C2;
3109 if (Cmp.isEquality() || !match(Y, m_APInt(C2)))
3110 return nullptr;
3111
3112 // Fold icmp pred (add X, C2), C.
3113 Type *Ty = Add->getType();
3114
3115 // If the add does not wrap, we can always adjust the compare by subtracting
3116 // the constants. Equality comparisons are handled elsewhere. SGE/SLE/UGE/ULE
3117 // are canonicalized to SGT/SLT/UGT/ULT.
3118 if ((Add->hasNoSignedWrap() &&
3119 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) ||
3120 (Add->hasNoUnsignedWrap() &&
3121 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT))) {
3122 bool Overflow;
3123 APInt NewC =
3124 Cmp.isSigned() ? C.ssub_ov(*C2, Overflow) : C.usub_ov(*C2, Overflow);
3125 // If there is overflow, the result must be true or false.
3126 // TODO: Can we assert there is no overflow because InstSimplify always
3127 // handles those cases?
3128 if (!Overflow)
3129 // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2)
3130 return new ICmpInst(Pred, X, ConstantInt::get(Ty, NewC));
3131 }
3132
3133 if (ICmpInst::isUnsigned(Pred) && Add->hasNoSignedWrap() &&
3134 C.isNonNegative() && (C - *C2).isNonNegative() &&
3135 computeConstantRange(X, /*ForSigned=*/true).add(*C2).isAllNonNegative())
3136 return new ICmpInst(ICmpInst::getSignedPredicate(Pred), X,
3137 ConstantInt::get(Ty, C - *C2));
3138
3139 auto CR = ConstantRange::makeExactICmpRegion(Pred, C).subtract(*C2);
3140 const APInt &Upper = CR.getUpper();
3141 const APInt &Lower = CR.getLower();
3142 if (Cmp.isSigned()) {
3143 if (Lower.isSignMask())
3144 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, Upper));
3145 if (Upper.isSignMask())
3146 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, Lower));
3147 } else {
3148 if (Lower.isMinValue())
3149 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, Upper));
3150 if (Upper.isMinValue())
3151 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, Lower));
3152 }
3153
3154 // This set of folds is intentionally placed after folds that use no-wrapping
3155 // flags because those folds are likely better for later analysis/codegen.
3158
3159 // Fold compare with offset to opposite sign compare if it eliminates offset:
3160 // (X + C2) >u C --> X <s -C2 (if C == C2 + SMAX)
3161 if (Pred == CmpInst::ICMP_UGT && C == *C2 + SMax)
3162 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, -(*C2)));
3163
3164 // (X + C2) <u C --> X >s ~C2 (if C == C2 + SMIN)
3165 if (Pred == CmpInst::ICMP_ULT && C == *C2 + SMin)
3166 return new ICmpInst(ICmpInst::ICMP_SGT, X, ConstantInt::get(Ty, ~(*C2)));
3167
3168 // (X + C2) >s C --> X <u (SMAX - C) (if C == C2 - 1)
3169 if (Pred == CmpInst::ICMP_SGT && C == *C2 - 1)
3170 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, SMax - C));
3171
3172 // (X + C2) <s C --> X >u (C ^ SMAX) (if C == C2)
3173 if (Pred == CmpInst::ICMP_SLT && C == *C2)
3174 return new ICmpInst(ICmpInst::ICMP_UGT, X, ConstantInt::get(Ty, C ^ SMax));
3175
3176 // (X + -1) <u C --> X <=u C (if X is never null)
3177 if (Pred == CmpInst::ICMP_ULT && C2->isAllOnes()) {
3178 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
3179 if (llvm::isKnownNonZero(X, Q))
3180 return new ICmpInst(ICmpInst::ICMP_ULE, X, ConstantInt::get(Ty, C));
3181 }
3182
3183 if (!Add->hasOneUse())
3184 return nullptr;
3185
3186 // X+C <u C2 -> (X & -C2) == C
3187 // iff C & (C2-1) == 0
3188 // C2 is a power of 2
3189 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && (*C2 & (C - 1)) == 0)
3191 ConstantExpr::getNeg(cast<Constant>(Y)));
3192
3193 // X+C2 <u C -> (X & C) == 2C
3194 // iff C == -(C2)
3195 // C2 is a power of 2
3196 if (Pred == ICmpInst::ICMP_ULT && C2->isPowerOf2() && C == -*C2)
3198 ConstantInt::get(Ty, C * 2));
3199
3200 // X+C >u C2 -> (X & ~C2) != C
3201 // iff C & C2 == 0
3202 // C2+1 is a power of 2
3203 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == 0)
3205 ConstantExpr::getNeg(cast<Constant>(Y)));
3206
3207 // The range test idiom can use either ult or ugt. Arbitrarily canonicalize
3208 // to the ult form.
3209 // X+C2 >u C -> X+(C2-C-1) <u ~C
3210 if (Pred == ICmpInst::ICMP_UGT)
3211 return new ICmpInst(ICmpInst::ICMP_ULT,
3212 Builder.CreateAdd(X, ConstantInt::get(Ty, *C2 - C - 1)),
3213 ConstantInt::get(Ty, ~C));
3214
3215 // zext(V) + C2 pred C -> V + C3 pred' C4
3216 Value *V;
3217 if (match(X, m_ZExt(m_Value(V)))) {
3218 Type *NewCmpTy = V->getType();
3219 unsigned NewCmpBW = NewCmpTy->getScalarSizeInBits();
3220 if (shouldChangeType(Ty, NewCmpTy)) {
3221 if (CR.getActiveBits() <= NewCmpBW) {
3222 ConstantRange SrcCR = CR.truncate(NewCmpBW);
3223 CmpInst::Predicate EquivPred;
3224 APInt EquivInt;
3225 APInt EquivOffset;
3226
3227 SrcCR.getEquivalentICmp(EquivPred, EquivInt, EquivOffset);
3228 return new ICmpInst(
3229 EquivPred,
3230 EquivOffset.isZero()
3231 ? V
3232 : Builder.CreateAdd(V, ConstantInt::get(NewCmpTy, EquivOffset)),
3233 ConstantInt::get(NewCmpTy, EquivInt));
3234 }
3235 }
3236 }
3237
3238 return nullptr;
3239}
3240
3242 Value *&RHS, ConstantInt *&Less,
3243 ConstantInt *&Equal,
3244 ConstantInt *&Greater) {
3245 // TODO: Generalize this to work with other comparison idioms or ensure
3246 // they get canonicalized into this form.
3247
3248 // select i1 (a == b),
3249 // i32 Equal,
3250 // i32 (select i1 (a < b), i32 Less, i32 Greater)
3251 // where Equal, Less and Greater are placeholders for any three constants.
3252 CmpPredicate PredA;
3253 if (!match(SI->getCondition(), m_ICmp(PredA, m_Value(LHS), m_Value(RHS))) ||
3254 !ICmpInst::isEquality(PredA))
3255 return false;
3256 Value *EqualVal = SI->getTrueValue();
3257 Value *UnequalVal = SI->getFalseValue();
3258 // We still can get non-canonical predicate here, so canonicalize.
3259 if (PredA == ICmpInst::ICMP_NE)
3260 std::swap(EqualVal, UnequalVal);
3261 if (!match(EqualVal, m_ConstantInt(Equal)))
3262 return false;
3263 CmpPredicate PredB;
3264 Value *LHS2, *RHS2;
3265 if (!match(UnequalVal, m_Select(m_ICmp(PredB, m_Value(LHS2), m_Value(RHS2)),
3266 m_ConstantInt(Less), m_ConstantInt(Greater))))
3267 return false;
3268 // We can get predicate mismatch here, so canonicalize if possible:
3269 // First, ensure that 'LHS' match.
3270 if (LHS2 != LHS) {
3271 // x sgt y <--> y slt x
3272 std::swap(LHS2, RHS2);
3273 PredB = ICmpInst::getSwappedPredicate(PredB);
3274 }
3275 if (LHS2 != LHS)
3276 return false;
3277 // We also need to canonicalize 'RHS'.
3278 if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(RHS2)) {
3279 // x sgt C-1 <--> x sge C <--> not(x slt C)
3280 auto FlippedStrictness =
3282 PredB, cast<Constant>(RHS2));
3283 if (!FlippedStrictness)
3284 return false;
3285 assert(FlippedStrictness->first == ICmpInst::ICMP_SGE &&
3286 "basic correctness failure");
3287 RHS2 = FlippedStrictness->second;
3288 // And kind-of perform the result swap.
3289 std::swap(Less, Greater);
3290 PredB = ICmpInst::ICMP_SLT;
3291 }
3292 return PredB == ICmpInst::ICMP_SLT && RHS == RHS2;
3293}
3294
3297 ConstantInt *C) {
3298
3299 assert(C && "Cmp RHS should be a constant int!");
3300 // If we're testing a constant value against the result of a three way
3301 // comparison, the result can be expressed directly in terms of the
3302 // original values being compared. Note: We could possibly be more
3303 // aggressive here and remove the hasOneUse test. The original select is
3304 // really likely to simplify or sink when we remove a test of the result.
3305 Value *OrigLHS, *OrigRHS;
3306 ConstantInt *C1LessThan, *C2Equal, *C3GreaterThan;
3307 if (Cmp.hasOneUse() &&
3308 matchThreeWayIntCompare(Select, OrigLHS, OrigRHS, C1LessThan, C2Equal,
3309 C3GreaterThan)) {
3310 assert(C1LessThan && C2Equal && C3GreaterThan);
3311
3312 bool TrueWhenLessThan = ICmpInst::compare(
3313 C1LessThan->getValue(), C->getValue(), Cmp.getPredicate());
3314 bool TrueWhenEqual = ICmpInst::compare(C2Equal->getValue(), C->getValue(),
3315 Cmp.getPredicate());
3316 bool TrueWhenGreaterThan = ICmpInst::compare(
3317 C3GreaterThan->getValue(), C->getValue(), Cmp.getPredicate());
3318
3319 // This generates the new instruction that will replace the original Cmp
3320 // Instruction. Instead of enumerating the various combinations when
3321 // TrueWhenLessThan, TrueWhenEqual and TrueWhenGreaterThan are true versus
3322 // false, we rely on chaining of ORs and future passes of InstCombine to
3323 // simplify the OR further (i.e. a s< b || a == b becomes a s<= b).
3324
3325 // When none of the three constants satisfy the predicate for the RHS (C),
3326 // the entire original Cmp can be simplified to a false.
3328 if (TrueWhenLessThan)
3330 OrigLHS, OrigRHS));
3331 if (TrueWhenEqual)
3333 OrigLHS, OrigRHS));
3334 if (TrueWhenGreaterThan)
3336 OrigLHS, OrigRHS));
3337
3338 return replaceInstUsesWith(Cmp, Cond);
3339 }
3340 return nullptr;
3341}
3342
3344 auto *Bitcast = dyn_cast<BitCastInst>(Cmp.getOperand(0));
3345 if (!Bitcast)
3346 return nullptr;
3347
3348 ICmpInst::Predicate Pred = Cmp.getPredicate();
3349 Value *Op1 = Cmp.getOperand(1);
3350 Value *BCSrcOp = Bitcast->getOperand(0);
3351 Type *SrcType = Bitcast->getSrcTy();
3352 Type *DstType = Bitcast->getType();
3353
3354 // Make sure the bitcast doesn't change between scalar and vector and
3355 // doesn't change the number of vector elements.
3356 if (SrcType->isVectorTy() == DstType->isVectorTy() &&
3357 SrcType->getScalarSizeInBits() == DstType->getScalarSizeInBits()) {
3358 // Zero-equality and sign-bit checks are preserved through sitofp + bitcast.
3359 Value *X;
3360 if (match(BCSrcOp, m_SIToFP(m_Value(X)))) {
3361 // icmp eq (bitcast (sitofp X)), 0 --> icmp eq X, 0
3362 // icmp ne (bitcast (sitofp X)), 0 --> icmp ne X, 0
3363 // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0
3364 // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0
3365 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT ||
3366 Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) &&
3367 match(Op1, m_Zero()))
3368 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
3369
3370 // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1
3371 if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_One()))
3372 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), 1));
3373
3374 // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1
3375 if (Pred == ICmpInst::ICMP_SGT && match(Op1, m_AllOnes()))
3376 return new ICmpInst(Pred, X,
3377 ConstantInt::getAllOnesValue(X->getType()));
3378 }
3379
3380 // Zero-equality checks are preserved through unsigned floating-point casts:
3381 // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0
3382 // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0
3383 if (match(BCSrcOp, m_UIToFP(m_Value(X))))
3384 if (Cmp.isEquality() && match(Op1, m_Zero()))
3385 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
3386
3387 const APInt *C;
3388 bool TrueIfSigned;
3389 if (match(Op1, m_APInt(C)) && Bitcast->hasOneUse()) {
3390 // If this is a sign-bit test of a bitcast of a casted FP value, eliminate
3391 // the FP extend/truncate because that cast does not change the sign-bit.
3392 // This is true for all standard IEEE-754 types and the X86 80-bit type.
3393 // The sign-bit is always the most significant bit in those types.
3394 if (isSignBitCheck(Pred, *C, TrueIfSigned) &&
3395 (match(BCSrcOp, m_FPExt(m_Value(X))) ||
3396 match(BCSrcOp, m_FPTrunc(m_Value(X))))) {
3397 // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0
3398 // (bitcast (fpext/fptrunc X)) to iX) > -1 --> (bitcast X to iY) > -1
3399 Type *XType = X->getType();
3400
3401 // We can't currently handle Power style floating point operations here.
3402 if (!(XType->isPPC_FP128Ty() || SrcType->isPPC_FP128Ty())) {
3403 Type *NewType = Builder.getIntNTy(XType->getScalarSizeInBits());
3404 if (auto *XVTy = dyn_cast<VectorType>(XType))
3405 NewType = VectorType::get(NewType, XVTy->getElementCount());
3406 Value *NewBitcast = Builder.CreateBitCast(X, NewType);
3407 if (TrueIfSigned)
3408 return new ICmpInst(ICmpInst::ICMP_SLT, NewBitcast,
3409 ConstantInt::getNullValue(NewType));
3410 else
3411 return new ICmpInst(ICmpInst::ICMP_SGT, NewBitcast,
3413 }
3414 }
3415
3416 // icmp eq/ne (bitcast X to int), special fp -> llvm.is.fpclass(X, class)
3417 Type *FPType = SrcType->getScalarType();
3418 if (!Cmp.getParent()->getParent()->hasFnAttribute(
3419 Attribute::NoImplicitFloat) &&
3420 Cmp.isEquality() && FPType->isIEEELikeFPTy()) {
3421 FPClassTest Mask = APFloat(FPType->getFltSemantics(), *C).classify();
3422 if (Mask & (fcInf | fcZero)) {
3423 if (Pred == ICmpInst::ICMP_NE)
3424 Mask = ~Mask;
3425 return replaceInstUsesWith(Cmp,
3426 Builder.createIsFPClass(BCSrcOp, Mask));
3427 }
3428 }
3429 }
3430 }
3431
3432 const APInt *C;
3433 if (!match(Cmp.getOperand(1), m_APInt(C)) || !DstType->isIntegerTy() ||
3434 !SrcType->isIntOrIntVectorTy())
3435 return nullptr;
3436
3437 // If this is checking if all elements of a vector compare are set or not,
3438 // invert the casted vector equality compare and test if all compare
3439 // elements are clear or not. Compare against zero is generally easier for
3440 // analysis and codegen.
3441 // icmp eq/ne (bitcast (not X) to iN), -1 --> icmp eq/ne (bitcast X to iN), 0
3442 // Example: are all elements equal? --> are zero elements not equal?
3443 // TODO: Try harder to reduce compare of 2 freely invertible operands?
3444 if (Cmp.isEquality() && C->isAllOnes() && Bitcast->hasOneUse()) {
3445 if (Value *NotBCSrcOp =
3446 getFreelyInverted(BCSrcOp, BCSrcOp->hasOneUse(), &Builder)) {
3447 Value *Cast = Builder.CreateBitCast(NotBCSrcOp, DstType);
3448 return new ICmpInst(Pred, Cast, ConstantInt::getNullValue(DstType));
3449 }
3450 }
3451
3452 // If this is checking if all elements of an extended vector are clear or not,
3453 // compare in a narrow type to eliminate the extend:
3454 // icmp eq/ne (bitcast (ext X) to iN), 0 --> icmp eq/ne (bitcast X to iM), 0
3455 Value *X;
3456 if (Cmp.isEquality() && C->isZero() && Bitcast->hasOneUse() &&
3457 match(BCSrcOp, m_ZExtOrSExt(m_Value(X)))) {
3458 if (auto *VecTy = dyn_cast<FixedVectorType>(X->getType())) {
3459 Type *NewType = Builder.getIntNTy(VecTy->getPrimitiveSizeInBits());
3460 Value *NewCast = Builder.CreateBitCast(X, NewType);
3461 return new ICmpInst(Pred, NewCast, ConstantInt::getNullValue(NewType));
3462 }
3463 }
3464
3465 // Folding: icmp <pred> iN X, C
3466 // where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN
3467 // and C is a splat of a K-bit pattern
3468 // and SC is a constant vector = <C', C', C', ..., C'>
3469 // Into:
3470 // %E = extractelement <M x iK> %vec, i32 C'
3471 // icmp <pred> iK %E, trunc(C)
3472 Value *Vec;
3473 ArrayRef<int> Mask;
3474 if (match(BCSrcOp, m_Shuffle(m_Value(Vec), m_Undef(), m_Mask(Mask)))) {
3475 // Check whether every element of Mask is the same constant
3476 if (all_equal(Mask)) {
3477 auto *VecTy = cast<VectorType>(SrcType);
3478 auto *EltTy = cast<IntegerType>(VecTy->getElementType());
3479 if (C->isSplat(EltTy->getBitWidth())) {
3480 // Fold the icmp based on the value of C
3481 // If C is M copies of an iK sized bit pattern,
3482 // then:
3483 // => %E = extractelement <N x iK> %vec, i32 Elem
3484 // icmp <pred> iK %SplatVal, <pattern>
3485 Value *Elem = Builder.getInt32(Mask[0]);
3486 Value *Extract = Builder.CreateExtractElement(Vec, Elem);
3487 Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth()));
3488 return new ICmpInst(Pred, Extract, NewC);
3489 }
3490 }
3491 }
3492 return nullptr;
3493}
3494
3495/// Try to fold integer comparisons with a constant operand: icmp Pred X, C
3496/// where X is some kind of instruction.
3498 const APInt *C;
3499
3500 if (match(Cmp.getOperand(1), m_APInt(C))) {
3501 if (auto *BO = dyn_cast<BinaryOperator>(Cmp.getOperand(0)))
3502 if (Instruction *I = foldICmpBinOpWithConstant(Cmp, BO, *C))
3503 return I;
3504
3505 if (auto *SI = dyn_cast<SelectInst>(Cmp.getOperand(0)))
3506 // For now, we only support constant integers while folding the
3507 // ICMP(SELECT)) pattern. We can extend this to support vector of integers
3508 // similar to the cases handled by binary ops above.
3509 if (auto *ConstRHS = dyn_cast<ConstantInt>(Cmp.getOperand(1)))
3510 if (Instruction *I = foldICmpSelectConstant(Cmp, SI, ConstRHS))
3511 return I;
3512
3513 if (auto *TI = dyn_cast<TruncInst>(Cmp.getOperand(0)))
3514 if (Instruction *I = foldICmpTruncConstant(Cmp, TI, *C))
3515 return I;
3516
3517 if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0)))
3519 return I;
3520
3521 // (extractval ([s/u]subo X, Y), 0) == 0 --> X == Y
3522 // (extractval ([s/u]subo X, Y), 0) != 0 --> X != Y
3523 // TODO: This checks one-use, but that is not strictly necessary.
3524 Value *Cmp0 = Cmp.getOperand(0);
3525 Value *X, *Y;
3526 if (C->isZero() && Cmp.isEquality() && Cmp0->hasOneUse() &&
3527 (match(Cmp0,
3528 m_ExtractValue<0>(m_Intrinsic<Intrinsic::ssub_with_overflow>(
3529 m_Value(X), m_Value(Y)))) ||
3530 match(Cmp0,
3531 m_ExtractValue<0>(m_Intrinsic<Intrinsic::usub_with_overflow>(
3532 m_Value(X), m_Value(Y))))))
3533 return new ICmpInst(Cmp.getPredicate(), X, Y);
3534 }
3535
3536 if (match(Cmp.getOperand(1), m_APIntAllowPoison(C)))
3538
3539 return nullptr;
3540}
3541
3542/// Fold an icmp equality instruction with binary operator LHS and constant RHS:
3543/// icmp eq/ne BO, C.
3545 ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) {
3546 // TODO: Some of these folds could work with arbitrary constants, but this
3547 // function is limited to scalar and vector splat constants.
3548 if (!Cmp.isEquality())
3549 return nullptr;
3550
3551 ICmpInst::Predicate Pred = Cmp.getPredicate();
3552 bool isICMP_NE = Pred == ICmpInst::ICMP_NE;
3553 Constant *RHS = cast<Constant>(Cmp.getOperand(1));
3554 Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1);
3555
3556 switch (BO->getOpcode()) {
3557 case Instruction::SRem:
3558 // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one.
3559 if (C.isZero() && BO->hasOneUse()) {
3560 const APInt *BOC;
3561 if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) {
3562 Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName());
3563 return new ICmpInst(Pred, NewRem,
3565 }
3566 }
3567 break;
3568 case Instruction::Add: {
3569 // (A + C2) == C --> A == (C - C2)
3570 // (A + C2) != C --> A != (C - C2)
3571 // TODO: Remove the one-use limitation? See discussion in D58633.
3572 if (Constant *C2 = dyn_cast<Constant>(BOp1)) {
3573 if (BO->hasOneUse())
3574 return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(RHS, C2));
3575 } else if (C.isZero()) {
3576 // Replace ((add A, B) != 0) with (A != -B) if A or B is
3577 // efficiently invertible, or if the add has just this one use.
3578 if (Value *NegVal = dyn_castNegVal(BOp1))
3579 return new ICmpInst(Pred, BOp0, NegVal);
3580 if (Value *NegVal = dyn_castNegVal(BOp0))
3581 return new ICmpInst(Pred, NegVal, BOp1);
3582 if (BO->hasOneUse()) {
3583 // (add nuw A, B) != 0 -> (or A, B) != 0
3584 if (match(BO, m_NUWAdd(m_Value(), m_Value()))) {
3585 Value *Or = Builder.CreateOr(BOp0, BOp1);
3586 return new ICmpInst(Pred, Or, Constant::getNullValue(BO->getType()));
3587 }
3588 Value *Neg = Builder.CreateNeg(BOp1);
3589 Neg->takeName(BO);
3590 return new ICmpInst(Pred, BOp0, Neg);
3591 }
3592 }
3593 break;
3594 }
3595 case Instruction::Xor:
3596 if (Constant *BOC = dyn_cast<Constant>(BOp1)) {
3597 // For the xor case, we can xor two constants together, eliminating
3598 // the explicit xor.
3599 return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC));
3600 } else if (C.isZero()) {
3601 // Replace ((xor A, B) != 0) with (A != B)
3602 return new ICmpInst(Pred, BOp0, BOp1);
3603 }
3604 break;
3605 case Instruction::Or: {
3606 const APInt *BOC;
3607 if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) {
3608 // Comparing if all bits outside of a constant mask are set?
3609 // Replace (X | C) == -1 with (X & ~C) == ~C.
3610 // This removes the -1 constant.
3611 Constant *NotBOC = ConstantExpr::getNot(cast<Constant>(BOp1));
3612 Value *And = Builder.CreateAnd(BOp0, NotBOC);
3613 return new ICmpInst(Pred, And, NotBOC);
3614 }
3615 // (icmp eq (or (select cond, 0, NonZero), Other), 0)
3616 // -> (and cond, (icmp eq Other, 0))
3617 // (icmp ne (or (select cond, NonZero, 0), Other), 0)
3618 // -> (or cond, (icmp ne Other, 0))
3619 Value *Cond, *TV, *FV, *Other, *Sel;
3620 if (C.isZero() &&
3621 match(BO,
3624 m_Value(FV))),
3625 m_Value(Other)))) &&
3626 Cond->getType() == Cmp.getType()) {
3627 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
3628 // Easy case is if eq/ne matches whether 0 is trueval/falseval.
3629 if (Pred == ICmpInst::ICMP_EQ
3630 ? (match(TV, m_Zero()) && isKnownNonZero(FV, Q))
3631 : (match(FV, m_Zero()) && isKnownNonZero(TV, Q))) {
3632 Value *Cmp = Builder.CreateICmp(
3633 Pred, Other, Constant::getNullValue(Other->getType()));
3635 Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or, Cmp,
3636 Cond);
3637 }
3638 // Harder case is if eq/ne matches whether 0 is falseval/trueval. In this
3639 // case we need to invert the select condition so we need to be careful to
3640 // avoid creating extra instructions.
3641 // (icmp ne (or (select cond, 0, NonZero), Other), 0)
3642 // -> (or (not cond), (icmp ne Other, 0))
3643 // (icmp eq (or (select cond, NonZero, 0), Other), 0)
3644 // -> (and (not cond), (icmp eq Other, 0))
3645 //
3646 // Only do this if the inner select has one use, in which case we are
3647 // replacing `select` with `(not cond)`. Otherwise, we will create more
3648 // uses. NB: Trying to freely invert cond doesn't make sense here, as if
3649 // cond was freely invertable, the select arms would have been inverted.
3650 if (Sel->hasOneUse() &&
3651 (Pred == ICmpInst::ICMP_EQ
3652 ? (match(FV, m_Zero()) && isKnownNonZero(TV, Q))
3653 : (match(TV, m_Zero()) && isKnownNonZero(FV, Q)))) {
3654 Value *NotCond = Builder.CreateNot(Cond);
3655 Value *Cmp = Builder.CreateICmp(
3656 Pred, Other, Constant::getNullValue(Other->getType()));
3658 Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or, Cmp,
3659 NotCond);
3660 }
3661 }
3662 break;
3663 }
3664 case Instruction::UDiv:
3665 case Instruction::SDiv:
3666 if (BO->isExact()) {
3667 // div exact X, Y eq/ne 0 -> X eq/ne 0
3668 // div exact X, Y eq/ne 1 -> X eq/ne Y
3669 // div exact X, Y eq/ne C ->
3670 // if Y * C never-overflow && OneUse:
3671 // -> Y * C eq/ne X
3672 if (C.isZero())
3673 return new ICmpInst(Pred, BOp0, Constant::getNullValue(BO->getType()));
3674 else if (C.isOne())
3675 return new ICmpInst(Pred, BOp0, BOp1);
3676 else if (BO->hasOneUse()) {
3678 Instruction::Mul, BO->getOpcode() == Instruction::SDiv, BOp1,
3679 Cmp.getOperand(1), BO);
3681 Value *YC =
3682 Builder.CreateMul(BOp1, ConstantInt::get(BO->getType(), C));
3683 return new ICmpInst(Pred, YC, BOp0);
3684 }
3685 }
3686 }
3687 if (BO->getOpcode() == Instruction::UDiv && C.isZero()) {
3688 // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A)
3689 auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT;
3690 return new ICmpInst(NewPred, BOp1, BOp0);
3691 }
3692 break;
3693 default:
3694 break;
3695 }
3696 return nullptr;
3697}
3698
3700 const APInt &CRhs,
3701 InstCombiner::BuilderTy &Builder,
3702 const SimplifyQuery &Q) {
3703 assert(CtpopLhs->getIntrinsicID() == Intrinsic::ctpop &&
3704 "Non-ctpop intrin in ctpop fold");
3705 if (!CtpopLhs->hasOneUse())
3706 return nullptr;
3707
3708 // Power of 2 test:
3709 // isPow2OrZero : ctpop(X) u< 2
3710 // isPow2 : ctpop(X) == 1
3711 // NotPow2OrZero: ctpop(X) u> 1
3712 // NotPow2 : ctpop(X) != 1
3713 // If we know any bit of X can be folded to:
3714 // IsPow2 : X & (~Bit) == 0
3715 // NotPow2 : X & (~Bit) != 0
3716 const ICmpInst::Predicate Pred = I.getPredicate();
3717 if (((I.isEquality() || Pred == ICmpInst::ICMP_UGT) && CRhs == 1) ||
3718 (Pred == ICmpInst::ICMP_ULT && CRhs == 2)) {
3719 Value *Op = CtpopLhs->getArgOperand(0);
3720 KnownBits OpKnown = computeKnownBits(Op, Q.DL,
3721 /*Depth*/ 0, Q.AC, Q.CxtI, Q.DT);
3722 // No need to check for count > 1, that should be already constant folded.
3723 if (OpKnown.countMinPopulation() == 1) {
3724 Value *And = Builder.CreateAnd(
3725 Op, Constant::getIntegerValue(Op->getType(), ~(OpKnown.One)));
3726 return new ICmpInst(
3727 (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_ULT)
3730 And, Constant::getNullValue(Op->getType()));
3731 }
3732 }
3733
3734 return nullptr;
3735}
3736
3737/// Fold an equality icmp with LLVM intrinsic and constant operand.
3739 ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) {
3740 Type *Ty = II->getType();
3741 unsigned BitWidth = C.getBitWidth();
3742 const ICmpInst::Predicate Pred = Cmp.getPredicate();
3743
3744 switch (II->getIntrinsicID()) {
3745 case Intrinsic::abs:
3746 // abs(A) == 0 -> A == 0
3747 // abs(A) == INT_MIN -> A == INT_MIN
3748 if (C.isZero() || C.isMinSignedValue())
3749 return new ICmpInst(Pred, II->getArgOperand(0), ConstantInt::get(Ty, C));
3750 break;
3751
3752 case Intrinsic::bswap:
3753 // bswap(A) == C -> A == bswap(C)
3754 return new ICmpInst(Pred, II->getArgOperand(0),
3755 ConstantInt::get(Ty, C.byteSwap()));
3756
3757 case Intrinsic::bitreverse:
3758 // bitreverse(A) == C -> A == bitreverse(C)
3759 return new ICmpInst(Pred, II->getArgOperand(0),
3760 ConstantInt::get(Ty, C.reverseBits()));
3761
3762 case Intrinsic::ctlz:
3763 case Intrinsic::cttz: {
3764 // ctz(A) == bitwidth(A) -> A == 0 and likewise for !=
3765 if (C == BitWidth)
3766 return new ICmpInst(Pred, II->getArgOperand(0),
3768
3769 // ctz(A) == C -> A & Mask1 == Mask2, where Mask2 only has bit C set
3770 // and Mask1 has bits 0..C+1 set. Similar for ctl, but for high bits.
3771 // Limit to one use to ensure we don't increase instruction count.
3772 unsigned Num = C.getLimitedValue(BitWidth);
3773 if (Num != BitWidth && II->hasOneUse()) {
3774 bool IsTrailing = II->getIntrinsicID() == Intrinsic::cttz;
3775 APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(BitWidth, Num + 1)
3776 : APInt::getHighBitsSet(BitWidth, Num + 1);
3777 APInt Mask2 = IsTrailing
3780 return new ICmpInst(Pred, Builder.CreateAnd(II->getArgOperand(0), Mask1),
3781 ConstantInt::get(Ty, Mask2));
3782 }
3783 break;
3784 }
3785
3786 case Intrinsic::ctpop: {
3787 // popcount(A) == 0 -> A == 0 and likewise for !=
3788 // popcount(A) == bitwidth(A) -> A == -1 and likewise for !=
3789 bool IsZero = C.isZero();
3790 if (IsZero || C == BitWidth)
3791 return new ICmpInst(Pred, II->getArgOperand(0),
3792 IsZero ? Constant::getNullValue(Ty)
3794
3795 break;
3796 }
3797
3798 case Intrinsic::fshl:
3799 case Intrinsic::fshr:
3800 if (II->getArgOperand(0) == II->getArgOperand(1)) {
3801 const APInt *RotAmtC;
3802 // ror(X, RotAmtC) == C --> X == rol(C, RotAmtC)
3803 // rol(X, RotAmtC) == C --> X == ror(C, RotAmtC)
3804 if (match(II->getArgOperand(2), m_APInt(RotAmtC)))
3805 return new ICmpInst(Pred, II->getArgOperand(0),
3806 II->getIntrinsicID() == Intrinsic::fshl
3807 ? ConstantInt::get(Ty, C.rotr(*RotAmtC))
3808 : ConstantInt::get(Ty, C.rotl(*RotAmtC)));
3809 }
3810 break;
3811
3812 case Intrinsic::umax:
3813 case Intrinsic::uadd_sat: {
3814 // uadd.sat(a, b) == 0 -> (a | b) == 0
3815 // umax(a, b) == 0 -> (a | b) == 0
3816 if (C.isZero() && II->hasOneUse()) {
3817 Value *Or = Builder.CreateOr(II->getArgOperand(0), II->getArgOperand(1));
3818 return new ICmpInst(Pred, Or, Constant::getNullValue(Ty));
3819 }
3820 break;
3821 }
3822
3823 case Intrinsic::ssub_sat:
3824 // ssub.sat(a, b) == 0 -> a == b
3825 if (C.isZero())
3826 return new ICmpInst(Pred, II->getArgOperand(0), II->getArgOperand(1));
3827 break;
3828 case Intrinsic::usub_sat: {
3829 // usub.sat(a, b) == 0 -> a <= b
3830 if (C.isZero()) {
3831 ICmpInst::Predicate NewPred =
3833 return new ICmpInst(NewPred, II->getArgOperand(0), II->getArgOperand(1));
3834 }
3835 break;
3836 }
3837 default:
3838 break;
3839 }
3840
3841 return nullptr;
3842}
3843
3844/// Fold an icmp with LLVM intrinsics
3845static Instruction *
3847 InstCombiner::BuilderTy &Builder) {
3848 assert(Cmp.isEquality());
3849
3850 ICmpInst::Predicate Pred = Cmp.getPredicate();
3851 Value *Op0 = Cmp.getOperand(0);
3852 Value *Op1 = Cmp.getOperand(1);
3853 const auto *IIOp0 = dyn_cast<IntrinsicInst>(Op0);
3854 const auto *IIOp1 = dyn_cast<IntrinsicInst>(Op1);
3855 if (!IIOp0 || !IIOp1 || IIOp0->getIntrinsicID() != IIOp1->getIntrinsicID())
3856 return nullptr;
3857
3858 switch (IIOp0->getIntrinsicID()) {
3859 case Intrinsic::bswap:
3860 case Intrinsic::bitreverse:
3861 // If both operands are byte-swapped or bit-reversed, just compare the
3862 // original values.
3863 return new ICmpInst(Pred, IIOp0->getOperand(0), IIOp1->getOperand(0));
3864 case Intrinsic::fshl:
3865 case Intrinsic::fshr: {
3866 // If both operands are rotated by same amount, just compare the
3867 // original values.
3868 if (IIOp0->getOperand(0) != IIOp0->getOperand(1))
3869 break;
3870 if (IIOp1->getOperand(0) != IIOp1->getOperand(1))
3871 break;
3872 if (IIOp0->getOperand(2) == IIOp1->getOperand(2))
3873 return new ICmpInst(Pred, IIOp0->getOperand(0), IIOp1->getOperand(0));
3874
3875 // rotate(X, AmtX) == rotate(Y, AmtY)
3876 // -> rotate(X, AmtX - AmtY) == Y
3877 // Do this if either both rotates have one use or if only one has one use
3878 // and AmtX/AmtY are constants.
3879 unsigned OneUses = IIOp0->hasOneUse() + IIOp1->hasOneUse();
3880 if (OneUses == 2 ||
3881 (OneUses == 1 && match(IIOp0->getOperand(2), m_ImmConstant()) &&
3882 match(IIOp1->getOperand(2), m_ImmConstant()))) {
3883 Value *SubAmt =
3884 Builder.CreateSub(IIOp0->getOperand(2), IIOp1->getOperand(2));
3885 Value *CombinedRotate = Builder.CreateIntrinsic(
3886 Op0->getType(), IIOp0->getIntrinsicID(),
3887 {IIOp0->getOperand(0), IIOp0->getOperand(0), SubAmt});
3888 return new ICmpInst(Pred, IIOp1->getOperand(0), CombinedRotate);
3889 }
3890 } break;
3891 default:
3892 break;
3893 }
3894
3895 return nullptr;
3896}
3897
3898/// Try to fold integer comparisons with a constant operand: icmp Pred X, C
3899/// where X is some kind of instruction and C is AllowPoison.
3900/// TODO: Move more folds which allow poison to this function.
3903 const APInt &C) {
3904 const ICmpInst::Predicate Pred = Cmp.getPredicate();
3905 if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0))) {
3906 switch (II->getIntrinsicID()) {
3907 default:
3908 break;
3909 case Intrinsic::fshl:
3910 case Intrinsic::fshr:
3911 if (Cmp.isEquality() && II->getArgOperand(0) == II->getArgOperand(1)) {
3912 // (rot X, ?) == 0/-1 --> X == 0/-1
3913 if (C.isZero() || C.isAllOnes())
3914 return new ICmpInst(Pred, II->getArgOperand(0), Cmp.getOperand(1));
3915 }
3916 break;
3917 }
3918 }
3919
3920 return nullptr;
3921}
3922
3923/// Fold an icmp with BinaryOp and constant operand: icmp Pred BO, C.
3925 BinaryOperator *BO,
3926 const APInt &C) {
3927 switch (BO->getOpcode()) {
3928 case Instruction::Xor:
3929 if (Instruction *I = foldICmpXorConstant(Cmp, BO, C))
3930 return I;
3931 break;
3932 case Instruction::And:
3933 if (Instruction *I = foldICmpAndConstant(Cmp, BO, C))
3934 return I;
3935 break;
3936 case Instruction::Or:
3937 if (Instruction *I = foldICmpOrConstant(Cmp, BO, C))
3938 return I;
3939 break;
3940 case Instruction::Mul:
3941 if (Instruction *I = foldICmpMulConstant(Cmp, BO, C))
3942 return I;
3943 break;
3944 case Instruction::Shl:
3945 if (Instruction *I = foldICmpShlConstant(Cmp, BO, C))
3946 return I;
3947 break;
3948 case Instruction::LShr:
3949 case Instruction::AShr:
3950 if (Instruction *I = foldICmpShrConstant(Cmp, BO, C))
3951 return I;
3952 break;
3953 case Instruction::SRem:
3954 if (Instruction *I = foldICmpSRemConstant(Cmp, BO, C))
3955 return I;
3956 break;
3957 case Instruction::UDiv:
3958 if (Instruction *I = foldICmpUDivConstant(Cmp, BO, C))
3959 return I;
3960 [[fallthrough]];
3961 case Instruction::SDiv:
3962 if (Instruction *I = foldICmpDivConstant(Cmp, BO, C))
3963 return I;
3964 break;
3965 case Instruction::Sub:
3966 if (Instruction *I = foldICmpSubConstant(Cmp, BO, C))
3967 return I;
3968 break;
3969 case Instruction::Add:
3970 if (Instruction *I = foldICmpAddConstant(Cmp, BO, C))
3971 return I;
3972 break;
3973 default:
3974 break;
3975 }
3976
3977 // TODO: These folds could be refactored to be part of the above calls.
3978 return foldICmpBinOpEqualityWithConstant(Cmp, BO, C);
3979}
3980
3981static Instruction *
3983 const APInt &C,
3984 InstCombiner::BuilderTy &Builder) {
3985 // This transform may end up producing more than one instruction for the
3986 // intrinsic, so limit it to one user of the intrinsic.
3987 if (!II->hasOneUse())
3988 return nullptr;
3989
3990 // Let Y = [add/sub]_sat(X, C) pred C2
3991 // SatVal = The saturating value for the operation
3992 // WillWrap = Whether or not the operation will underflow / overflow
3993 // => Y = (WillWrap ? SatVal : (X binop C)) pred C2
3994 // => Y = WillWrap ? (SatVal pred C2) : ((X binop C) pred C2)
3995 //
3996 // When (SatVal pred C2) is true, then
3997 // Y = WillWrap ? true : ((X binop C) pred C2)
3998 // => Y = WillWrap || ((X binop C) pred C2)
3999 // else
4000 // Y = WillWrap ? false : ((X binop C) pred C2)
4001 // => Y = !WillWrap ? ((X binop C) pred C2) : false
4002 // => Y = !WillWrap && ((X binop C) pred C2)
4003 Value *Op0 = II->getOperand(0);
4004 Value *Op1 = II->getOperand(1);
4005
4006 const APInt *COp1;
4007 // This transform only works when the intrinsic has an integral constant or
4008 // splat vector as the second operand.
4009 if (!match(Op1, m_APInt(COp1)))
4010 return nullptr;
4011
4012 APInt SatVal;
4013 switch (II->getIntrinsicID()) {
4014 default:
4016 "This function only works with usub_sat and uadd_sat for now!");
4017 case Intrinsic::uadd_sat:
4018 SatVal = APInt::getAllOnes(C.getBitWidth());
4019 break;
4020 case Intrinsic::usub_sat:
4021 SatVal = APInt::getZero(C.getBitWidth());
4022 break;
4023 }
4024
4025 // Check (SatVal pred C2)
4026 bool SatValCheck = ICmpInst::compare(SatVal, C, Pred);
4027
4028 // !WillWrap.
4030 II->getBinaryOp(), *COp1, II->getNoWrapKind());
4031
4032 // WillWrap.
4033 if (SatValCheck)
4034 C1 = C1.inverse();
4035
4037 if (II->getBinaryOp() == Instruction::Add)
4038 C2 = C2.sub(*COp1);
4039 else
4040 C2 = C2.add(*COp1);
4041
4042 Instruction::BinaryOps CombiningOp =
4043 SatValCheck ? Instruction::BinaryOps::Or : Instruction::BinaryOps::And;
4044
4045 std::optional<ConstantRange> Combination;
4046 if (CombiningOp == Instruction::BinaryOps::Or)
4047 Combination = C1.exactUnionWith(C2);
4048 else /* CombiningOp == Instruction::BinaryOps::And */
4049 Combination = C1.exactIntersectWith(C2);
4050
4051 if (!Combination)
4052 return nullptr;
4053
4054 CmpInst::Predicate EquivPred;
4055 APInt EquivInt;
4056 APInt EquivOffset;
4057
4058 Combination->getEquivalentICmp(EquivPred, EquivInt, EquivOffset);
4059
4060 return new ICmpInst(
4061 EquivPred,
4062 Builder.CreateAdd(Op0, ConstantInt::get(Op1->getType(), EquivOffset)),
4063 ConstantInt::get(Op1->getType(), EquivInt));
4064}
4065
4066static Instruction *
4068 const APInt &C,
4069 InstCombiner::BuilderTy &Builder) {
4070 std::optional<ICmpInst::Predicate> NewPredicate = std::nullopt;
4071 switch (Pred) {
4072 case ICmpInst::ICMP_EQ:
4073 case ICmpInst::ICMP_NE:
4074 if (C.isZero())
4075 NewPredicate = Pred;
4076 else if (C.isOne())
4077 NewPredicate =
4079 else if (C.isAllOnes())
4080 NewPredicate =
4082 break;
4083
4084 case ICmpInst::ICMP_SGT:
4085 if (C.isAllOnes())
4086 NewPredicate = ICmpInst::ICMP_UGE;
4087 else if (C.isZero())
4088 NewPredicate = ICmpInst::ICMP_UGT;
4089 break;
4090
4091 case ICmpInst::ICMP_SLT:
4092 if (C.isZero())
4093 NewPredicate = ICmpInst::ICMP_ULT;
4094 else if (C.isOne())
4095 NewPredicate = ICmpInst::ICMP_ULE;
4096 break;
4097
4098 case ICmpInst::ICMP_ULT:
4099 if (C.ugt(1))
4100 NewPredicate = ICmpInst::ICMP_UGE;
4101 break;
4102
4103 case ICmpInst::ICMP_UGT:
4104 if (!C.isZero() && !C.isAllOnes())
4105 NewPredicate = ICmpInst::ICMP_ULT;
4106 break;
4107
4108 default:
4109 break;
4110 }
4111
4112 if (!NewPredicate)
4113 return nullptr;
4114
4115 if (I->getIntrinsicID() == Intrinsic::scmp)
4116 NewPredicate = ICmpInst::getSignedPredicate(*NewPredicate);
4117 Value *LHS = I->getOperand(0);
4118 Value *RHS = I->getOperand(1);
4119 return new ICmpInst(*NewPredicate, LHS, RHS);
4120}
4121
4122/// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
4125 const APInt &C) {
4126 ICmpInst::Predicate Pred = Cmp.getPredicate();
4127
4128 // Handle folds that apply for any kind of icmp.
4129 switch (II->getIntrinsicID()) {
4130 default:
4131 break;
4132 case Intrinsic::uadd_sat:
4133 case Intrinsic::usub_sat:
4134 if (auto *Folded = foldICmpUSubSatOrUAddSatWithConstant(
4135 Pred, cast<SaturatingInst>(II), C, Builder))
4136 return Folded;
4137 break;
4138 case Intrinsic::ctpop: {
4139 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
4140 if (Instruction *R = foldCtpopPow2Test(Cmp, II, C, Builder, Q))
4141 return R;
4142 } break;
4143 case Intrinsic::scmp:
4144 case Intrinsic::ucmp:
4145 if (auto *Folded = foldICmpOfCmpIntrinsicWithConstant(Pred, II, C, Builder))
4146 return Folded;
4147 break;
4148 }
4149
4150 if (Cmp.isEquality())
4151 return foldICmpEqIntrinsicWithConstant(Cmp, II, C);
4152
4153 Type *Ty = II->getType();
4154 unsigned BitWidth = C.getBitWidth();
4155 switch (II->getIntrinsicID()) {
4156 case Intrinsic::ctpop: {
4157 // (ctpop X > BitWidth - 1) --> X == -1
4158 Value *X = II->getArgOperand(0);
4159 if (C == BitWidth - 1 && Pred == ICmpInst::ICMP_UGT)
4160 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, X,
4162 // (ctpop X < BitWidth) --> X != -1
4163 if (C == BitWidth && Pred == ICmpInst::ICMP_ULT)
4164 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, X,
4166 break;
4167 }
4168 case Intrinsic::ctlz: {
4169 // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000
4170 if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) {
4171 unsigned Num = C.getLimitedValue();
4172 APInt Limit = APInt::getOneBitSet(BitWidth, BitWidth - Num - 1);
4173 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT,
4174 II->getArgOperand(0), ConstantInt::get(Ty, Limit));
4175 }
4176
4177 // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111
4178 if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) {
4179 unsigned Num = C.getLimitedValue();
4181 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT,
4182 II->getArgOperand(0), ConstantInt::get(Ty, Limit));
4183 }
4184 break;
4185 }
4186 case Intrinsic::cttz: {
4187 // Limit to one use to ensure we don't increase instruction count.
4188 if (!II->hasOneUse())
4189 return nullptr;
4190
4191 // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0
4192 if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) {
4193 APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue() + 1);
4194 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ,
4195 Builder.CreateAnd(II->getArgOperand(0), Mask),
4197 }
4198
4199 // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0
4200 if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) {
4201 APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue());
4202 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE,
4203 Builder.CreateAnd(II->getArgOperand(0), Mask),
4205 }
4206 break;
4207 }
4208 case Intrinsic::ssub_sat:
4209 // ssub.sat(a, b) spred 0 -> a spred b
4210 if (ICmpInst::isSigned(Pred)) {
4211 if (C.isZero())
4212 return new ICmpInst(Pred, II->getArgOperand(0), II->getArgOperand(1));
4213 // X s<= 0 is cannonicalized to X s< 1
4214 if (Pred == ICmpInst::ICMP_SLT && C.isOne())
4215 return new ICmpInst(ICmpInst::ICMP_SLE, II->getArgOperand(0),
4216 II->getArgOperand(1));
4217 // X s>= 0 is cannonicalized to X s> -1
4218 if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
4219 return new ICmpInst(ICmpInst::ICMP_SGE, II->getArgOperand(0),
4220 II->getArgOperand(1));
4221 }
4222 break;
4223 default:
4224 break;
4225 }
4226
4227 return nullptr;
4228}
4229
4230/// Handle icmp with constant (but not simple integer constant) RHS.
4232 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
4233 Constant *RHSC = dyn_cast<Constant>(Op1);
4234 Instruction *LHSI = dyn_cast<Instruction>(Op0);
4235 if (!RHSC || !LHSI)
4236 return nullptr;
4237
4238 switch (LHSI->getOpcode()) {
4239 case Instruction::PHI:
4240 if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI)))
4241 return NV;
4242 break;
4243 case Instruction::IntToPtr:
4244 // icmp pred inttoptr(X), null -> icmp pred X, 0
4245 if (RHSC->isNullValue() &&
4246 DL.getIntPtrType(RHSC->getType()) == LHSI->getOperand(0)->getType())
4247 return new ICmpInst(
4248 I.getPredicate(), LHSI->getOperand(0),
4250 break;
4251
4252 case Instruction::Load:
4253 // Try to optimize things like "A[i] > 4" to index computations.
4254 if (GetElementPtrInst *GEP =
4255 dyn_cast<GetElementPtrInst>(LHSI->getOperand(0)))
4256 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)))
4257 if (Instruction *Res =
4258 foldCmpLoadFromIndexedGlobal(cast<LoadInst>(LHSI), GEP, GV, I))
4259 return Res;
4260 break;
4261 }
4262
4263 return nullptr;
4264}
4265
4267 Value *RHS, const ICmpInst &I) {
4268 // Try to fold the comparison into the select arms, which will cause the
4269 // select to be converted into a logical and/or.
4270 auto SimplifyOp = [&](Value *Op, bool SelectCondIsTrue) -> Value * {
4271 if (Value *Res = simplifyICmpInst(Pred, Op, RHS, SQ))
4272 return Res;
4273 if (std::optional<bool> Impl = isImpliedCondition(
4274 SI->getCondition(), Pred, Op, RHS, DL, SelectCondIsTrue))
4275 return ConstantInt::get(I.getType(), *Impl);
4276 return nullptr;
4277 };
4278
4279 ConstantInt *CI = nullptr;
4280 Value *Op1 = SimplifyOp(SI->getOperand(1), true);
4281 if (Op1)
4282 CI = dyn_cast<ConstantInt>(Op1);
4283
4284 Value *Op2 = SimplifyOp(SI->getOperand(2), false);
4285 if (Op2)
4286 CI = dyn_cast<ConstantInt>(Op2);
4287
4288 auto Simplifies = [&](Value *Op, unsigned Idx) {
4289 // A comparison of ucmp/scmp with a constant will fold into an icmp.
4290 const APInt *Dummy;
4291 return Op ||
4292 (isa<CmpIntrinsic>(SI->getOperand(Idx)) &&
4293 SI->getOperand(Idx)->hasOneUse() && match(RHS, m_APInt(Dummy)));
4294 };
4295
4296 // We only want to perform this transformation if it will not lead to
4297 // additional code. This is true if either both sides of the select
4298 // fold to a constant (in which case the icmp is replaced with a select
4299 // which will usually simplify) or this is the only user of the
4300 // select (in which case we are trading a select+icmp for a simpler
4301 // select+icmp) or all uses of the select can be replaced based on
4302 // dominance information ("Global cases").
4303 bool Transform = false;
4304 if (Op1 && Op2)
4305 Transform = true;
4306 else if (Simplifies(Op1, 1) || Simplifies(Op2, 2)) {
4307 // Local case
4308 if (SI->hasOneUse())
4309 Transform = true;
4310 // Global cases
4311 else if (CI && !CI->isZero())
4312 // When Op1 is constant try replacing select with second operand.
4313 // Otherwise Op2 is constant and try replacing select with first
4314 // operand.
4315 Transform = replacedSelectWithOperand(SI, &I, Op1 ? 2 : 1);
4316 }
4317 if (Transform) {
4318 if (!Op1)
4319 Op1 = Builder.CreateICmp(Pred, SI->getOperand(1), RHS, I.getName());
4320 if (!Op2)
4321 Op2 = Builder.CreateICmp(Pred, SI->getOperand(2), RHS, I.getName());
4322 return SelectInst::Create(SI->getOperand(0), Op1, Op2);
4323 }
4324
4325 return nullptr;
4326}
4327
4328// Returns whether V is a Mask ((X + 1) & X == 0) or ~Mask (-Pow2OrZero)
4329static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q,
4330 unsigned Depth = 0) {
4331 if (Not ? match(V, m_NegatedPower2OrZero()) : match(V, m_LowBitMaskOrZero()))
4332 return true;
4333 if (V->getType()->getScalarSizeInBits() == 1)
4334 return true;
4336 return false;
4337 Value *X;
4338 const Instruction *I = dyn_cast<Instruction>(V);
4339 if (!I)
4340 return false;
4341 switch (I->getOpcode()) {
4342 case Instruction::ZExt:
4343 // ZExt(Mask) is a Mask.
4344 return !Not && isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4345 case Instruction::SExt:
4346 // SExt(Mask) is a Mask.
4347 // SExt(~Mask) is a ~Mask.
4348 return isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4349 case Instruction::And:
4350 case Instruction::Or:
4351 // Mask0 | Mask1 is a Mask.
4352 // Mask0 & Mask1 is a Mask.
4353 // ~Mask0 | ~Mask1 is a ~Mask.
4354 // ~Mask0 & ~Mask1 is a ~Mask.
4355 return isMaskOrZero(I->getOperand(1), Not, Q, Depth) &&
4356 isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4357 case Instruction::Xor:
4358 if (match(V, m_Not(m_Value(X))))
4359 return isMaskOrZero(X, !Not, Q, Depth);
4360
4361 // (X ^ -X) is a ~Mask
4362 if (Not)
4363 return match(V, m_c_Xor(m_Value(X), m_Neg(m_Deferred(X))));
4364 // (X ^ (X - 1)) is a Mask
4365 else
4366 return match(V, m_c_Xor(m_Value(X), m_Add(m_Deferred(X), m_AllOnes())));
4367 case Instruction::Select:
4368 // c ? Mask0 : Mask1 is a Mask.
4369 return isMaskOrZero(I->getOperand(1), Not, Q, Depth) &&
4370 isMaskOrZero(I->getOperand(2), Not, Q, Depth);
4371 case Instruction::Shl:
4372 // (~Mask) << X is a ~Mask.
4373 return Not && isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4374 case Instruction::LShr:
4375 // Mask >> X is a Mask.
4376 return !Not && isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4377 case Instruction::AShr:
4378 // Mask s>> X is a Mask.
4379 // ~Mask s>> X is a ~Mask.
4380 return isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4381 case Instruction::Add:
4382 // Pow2 - 1 is a Mask.
4383 if (!Not && match(I->getOperand(1), m_AllOnes()))
4384 return isKnownToBeAPowerOfTwo(I->getOperand(0), Q.DL, /*OrZero*/ true,
4385 Depth, Q.AC, Q.CxtI, Q.DT);
4386 break;
4387 case Instruction::Sub:
4388 // -Pow2 is a ~Mask.
4389 if (Not && match(I->getOperand(0), m_Zero()))
4390 return isKnownToBeAPowerOfTwo(I->getOperand(1), Q.DL, /*OrZero*/ true,
4391 Depth, Q.AC, Q.CxtI, Q.DT);
4392 break;
4393 case Instruction::Call: {
4394 if (auto *II = dyn_cast<IntrinsicInst>(I)) {
4395 switch (II->getIntrinsicID()) {
4396 // min/max(Mask0, Mask1) is a Mask.
4397 // min/max(~Mask0, ~Mask1) is a ~Mask.
4398 case Intrinsic::umax:
4399 case Intrinsic::smax:
4400 case Intrinsic::umin:
4401 case Intrinsic::smin:
4402 return isMaskOrZero(II->getArgOperand(1), Not, Q, Depth) &&
4403 isMaskOrZero(II->getArgOperand(0), Not, Q, Depth);
4404
4405 // In the context of masks, bitreverse(Mask) == ~Mask
4406 case Intrinsic::bitreverse:
4407 return isMaskOrZero(II->getArgOperand(0), !Not, Q, Depth);
4408 default:
4409 break;
4410 }
4411 }
4412 break;
4413 }
4414 default:
4415 break;
4416 }
4417 return false;
4418}
4419
4420/// Some comparisons can be simplified.
4421/// In this case, we are looking for comparisons that look like
4422/// a check for a lossy truncation.
4423/// Folds:
4424/// icmp SrcPred (x & Mask), x to icmp DstPred x, Mask
4425/// icmp SrcPred (x & ~Mask), ~Mask to icmp DstPred x, ~Mask
4426/// icmp eq/ne (x & ~Mask), 0 to icmp DstPred x, Mask
4427/// icmp eq/ne (~x | Mask), -1 to icmp DstPred x, Mask
4428/// Where Mask is some pattern that produces all-ones in low bits:
4429/// (-1 >> y)
4430/// ((-1 << y) >> y) <- non-canonical, has extra uses
4431/// ~(-1 << y)
4432/// ((1 << y) + (-1)) <- non-canonical, has extra uses
4433/// The Mask can be a constant, too.
4434/// For some predicates, the operands are commutative.
4435/// For others, x can only be on a specific side.
4437 Value *Op1, const SimplifyQuery &Q,
4438 InstCombiner &IC) {
4439
4440 ICmpInst::Predicate DstPred;
4441 switch (Pred) {
4443 // x & Mask == x
4444 // x & ~Mask == 0
4445 // ~x | Mask == -1
4446 // -> x u<= Mask
4447 // x & ~Mask == ~Mask
4448 // -> ~Mask u<= x
4450 break;
4452 // x & Mask != x
4453 // x & ~Mask != 0
4454 // ~x | Mask != -1
4455 // -> x u> Mask
4456 // x & ~Mask != ~Mask
4457 // -> ~Mask u> x
4459 break;
4461 // x & Mask u< x
4462 // -> x u> Mask
4463 // x & ~Mask u< ~Mask
4464 // -> ~Mask u> x
4466 break;
4468 // x & Mask u>= x
4469 // -> x u<= Mask
4470 // x & ~Mask u>= ~Mask
4471 // -> ~Mask u<= x
4473 break;
4475 // x & Mask s< x [iff Mask s>= 0]
4476 // -> x s> Mask
4477 // x & ~Mask s< ~Mask [iff ~Mask != 0]
4478 // -> ~Mask s> x
4480 break;
4482 // x & Mask s>= x [iff Mask s>= 0]
4483 // -> x s<= Mask
4484 // x & ~Mask s>= ~Mask [iff ~Mask != 0]
4485 // -> ~Mask s<= x
4487 break;
4488 default:
4489 // We don't support sgt,sle
4490 // ult/ugt are simplified to true/false respectively.
4491 return nullptr;
4492 }
4493
4494 Value *X, *M;
4495 // Put search code in lambda for early positive returns.
4496 auto IsLowBitMask = [&]() {
4497 if (match(Op0, m_c_And(m_Specific(Op1), m_Value(M)))) {
4498 X = Op1;
4499 // Look for: x & Mask pred x
4500 if (isMaskOrZero(M, /*Not=*/false, Q)) {
4501 return !ICmpInst::isSigned(Pred) ||
4502 (match(M, m_NonNegative()) || isKnownNonNegative(M, Q));
4503 }
4504
4505 // Look for: x & ~Mask pred ~Mask
4506 if (isMaskOrZero(X, /*Not=*/true, Q)) {
4507 return !ICmpInst::isSigned(Pred) || isKnownNonZero(X, Q);
4508 }
4509 return false;
4510 }
4511 if (ICmpInst::isEquality(Pred) && match(Op1, m_AllOnes()) &&
4512 match(Op0, m_OneUse(m_Or(m_Value(X), m_Value(M))))) {
4513
4514 auto Check = [&]() {
4515 // Look for: ~x | Mask == -1
4516 if (isMaskOrZero(M, /*Not=*/false, Q)) {
4517 if (Value *NotX =
4518 IC.getFreelyInverted(X, X->hasOneUse(), &IC.Builder)) {
4519 X = NotX;
4520 return true;
4521 }
4522 }
4523 return false;
4524 };
4525 if (Check())
4526 return true;
4527 std::swap(X, M);
4528 return Check();
4529 }
4530 if (ICmpInst::isEquality(Pred) && match(Op1, m_Zero()) &&
4531 match(Op0, m_OneUse(m_And(m_Value(X), m_Value(M))))) {
4532 auto Check = [&]() {
4533 // Look for: x & ~Mask == 0
4534 if (isMaskOrZero(M, /*Not=*/true, Q)) {
4535 if (Value *NotM =
4536 IC.getFreelyInverted(M, M->hasOneUse(), &IC.Builder)) {
4537 M = NotM;
4538 return true;
4539 }
4540 }
4541 return false;
4542 };
4543 if (Check())
4544 return true;
4545 std::swap(X, M);
4546 return Check();
4547 }
4548 return false;
4549 };
4550
4551 if (!IsLowBitMask())
4552 return nullptr;
4553
4554 return IC.Builder.CreateICmp(DstPred, X, M);
4555}
4556
4557/// Some comparisons can be simplified.
4558/// In this case, we are looking for comparisons that look like
4559/// a check for a lossy signed truncation.
4560/// Folds: (MaskedBits is a constant.)
4561/// ((%x << MaskedBits) a>> MaskedBits) SrcPred %x
4562/// Into:
4563/// (add %x, (1 << (KeptBits-1))) DstPred (1 << KeptBits)
4564/// Where KeptBits = bitwidth(%x) - MaskedBits
4565static Value *
4567 InstCombiner::BuilderTy &Builder) {
4568 CmpPredicate SrcPred;
4569 Value *X;
4570 const APInt *C0, *C1; // FIXME: non-splats, potentially with undef.
4571 // We are ok with 'shl' having multiple uses, but 'ashr' must be one-use.
4572 if (!match(&I, m_c_ICmp(SrcPred,
4574 m_APInt(C1))),
4575 m_Deferred(X))))
4576 return nullptr;
4577
4578 // Potential handling of non-splats: for each element:
4579 // * if both are undef, replace with constant 0.
4580 // Because (1<<0) is OK and is 1, and ((1<<0)>>1) is also OK and is 0.
4581 // * if both are not undef, and are different, bailout.
4582 // * else, only one is undef, then pick the non-undef one.
4583
4584 // The shift amount must be equal.
4585 if (*C0 != *C1)
4586 return nullptr;
4587 const APInt &MaskedBits = *C0;
4588 assert(MaskedBits != 0 && "shift by zero should be folded away already.");
4589
4590 ICmpInst::Predicate DstPred;
4591 switch (SrcPred) {
4593 // ((%x << MaskedBits) a>> MaskedBits) == %x
4594 // =>
4595 // (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits)
4597 break;
4599 // ((%x << MaskedBits) a>> MaskedBits) != %x
4600 // =>
4601 // (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits)
4603 break;
4604 // FIXME: are more folds possible?
4605 default:
4606 return nullptr;
4607 }
4608
4609 auto *XType = X->getType();
4610 const unsigned XBitWidth = XType->getScalarSizeInBits();
4611 const APInt BitWidth = APInt(XBitWidth, XBitWidth);
4612 assert(BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched");
4613
4614 // KeptBits = bitwidth(%x) - MaskedBits
4615 const APInt KeptBits = BitWidth - MaskedBits;
4616 assert(KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable");
4617 // ICmpCst = (1 << KeptBits)
4618 const APInt ICmpCst = APInt(XBitWidth, 1).shl(KeptBits);
4619 assert(ICmpCst.isPowerOf2());
4620 // AddCst = (1 << (KeptBits-1))
4621 const APInt AddCst = ICmpCst.lshr(1);
4622 assert(AddCst.ult(ICmpCst) && AddCst.isPowerOf2());
4623
4624 // T0 = add %x, AddCst
4625 Value *T0 = Builder.CreateAdd(X, ConstantInt::get(XType, AddCst));
4626 // T1 = T0 DstPred ICmpCst
4627 Value *T1 = Builder.CreateICmp(DstPred, T0, ConstantInt::get(XType, ICmpCst));
4628
4629 return T1;
4630}
4631
4632// Given pattern:
4633// icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
4634// we should move shifts to the same hand of 'and', i.e. rewrite as
4635// icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x)
4636// We are only interested in opposite logical shifts here.
4637// One of the shifts can be truncated.
4638// If we can, we want to end up creating 'lshr' shift.
4639static Value *
4641 InstCombiner::BuilderTy &Builder) {
4642 if (!I.isEquality() || !match(I.getOperand(1), m_Zero()) ||
4643 !I.getOperand(0)->hasOneUse())
4644 return nullptr;
4645
4646 auto m_AnyLogicalShift = m_LogicalShift(m_Value(), m_Value());
4647
4648 // Look for an 'and' of two logical shifts, one of which may be truncated.
4649 // We use m_TruncOrSelf() on the RHS to correctly handle commutative case.
4650 Instruction *XShift, *MaybeTruncation, *YShift;
4651 if (!match(
4652 I.getOperand(0),
4653 m_c_And(m_CombineAnd(m_AnyLogicalShift, m_Instruction(XShift)),
4655 m_AnyLogicalShift, m_Instruction(YShift))),
4656 m_Instruction(MaybeTruncation)))))
4657 return nullptr;
4658
4659 // We potentially looked past 'trunc', but only when matching YShift,
4660 // therefore YShift must have the widest type.
4661 Instruction *WidestShift = YShift;
4662 // Therefore XShift must have the shallowest type.
4663 // Or they both have identical types if there was no truncation.
4664 Instruction *NarrowestShift = XShift;
4665
4666 Type *WidestTy = WidestShift->getType();
4667 Type *NarrowestTy = NarrowestShift->getType();
4668 assert(NarrowestTy == I.getOperand(0)->getType() &&
4669 "We did not look past any shifts while matching XShift though.");
4670 bool HadTrunc = WidestTy != I.getOperand(0)->getType();
4671
4672 // If YShift is a 'lshr', swap the shifts around.
4673 if (match(YShift, m_LShr(m_Value(), m_Value())))
4674 std::swap(XShift, YShift);
4675
4676 // The shifts must be in opposite directions.
4677 auto XShiftOpcode = XShift->getOpcode();
4678 if (XShiftOpcode == YShift->getOpcode())
4679 return nullptr; // Do not care about same-direction shifts here.
4680
4681 Value *X, *XShAmt, *Y, *YShAmt;
4682 match(XShift, m_BinOp(m_Value(X), m_ZExtOrSelf(m_Value(XShAmt))));
4683 match(YShift, m_BinOp(m_Value(Y), m_ZExtOrSelf(m_Value(YShAmt))));
4684
4685 // If one of the values being shifted is a constant, then we will end with
4686 // and+icmp, and [zext+]shift instrs will be constant-folded. If they are not,
4687 // however, we will need to ensure that we won't increase instruction count.
4688 if (!isa<Constant>(X) && !isa<Constant>(Y)) {
4689 // At least one of the hands of the 'and' should be one-use shift.
4690 if (!match(I.getOperand(0),
4691 m_c_And(m_OneUse(m_AnyLogicalShift), m_Value())))
4692 return nullptr;
4693 if (HadTrunc) {
4694 // Due to the 'trunc', we will need to widen X. For that either the old
4695 // 'trunc' or the shift amt in the non-truncated shift should be one-use.
4696 if (!MaybeTruncation->hasOneUse() &&
4697 !NarrowestShift->getOperand(1)->hasOneUse())
4698 return nullptr;
4699 }
4700 }
4701
4702 // We have two shift amounts from two different shifts. The types of those
4703 // shift amounts may not match. If that's the case let's bailout now.
4704 if (XShAmt->getType() != YShAmt->getType())
4705 return nullptr;
4706
4707 // As input, we have the following pattern:
4708 // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
4709 // We want to rewrite that as:
4710 // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x)
4711 // While we know that originally (Q+K) would not overflow
4712 // (because 2 * (N-1) u<= iN -1), we have looked past extensions of
4713 // shift amounts. so it may now overflow in smaller bitwidth.
4714 // To ensure that does not happen, we need to ensure that the total maximal
4715 // shift amount is still representable in that smaller bit width.
4716 unsigned MaximalPossibleTotalShiftAmount =
4717 (WidestTy->getScalarSizeInBits() - 1) +
4718 (NarrowestTy->getScalarSizeInBits() - 1);
4719 APInt MaximalRepresentableShiftAmount =
4721 if (MaximalRepresentableShiftAmount.ult(MaximalPossibleTotalShiftAmount))
4722 return nullptr;
4723
4724 // Can we fold (XShAmt+YShAmt) ?
4725 auto *NewShAmt = dyn_cast_or_null<Constant>(
4726 simplifyAddInst(XShAmt, YShAmt, /*isNSW=*/false,
4727 /*isNUW=*/false, SQ.getWithInstruction(&I)));
4728 if (!NewShAmt)
4729 return nullptr;
4730 if (NewShAmt->getType() != WidestTy) {
4731 NewShAmt =
4732 ConstantFoldCastOperand(Instruction::ZExt, NewShAmt, WidestTy, SQ.DL);
4733 if (!NewShAmt)
4734 return nullptr;
4735 }
4736 unsigned WidestBitWidth = WidestTy->getScalarSizeInBits();
4737
4738 // Is the new shift amount smaller than the bit width?
4739 // FIXME: could also rely on ConstantRange.
4740 if (!match(NewShAmt,
4742 APInt(WidestBitWidth, WidestBitWidth))))
4743 return nullptr;
4744
4745 // An extra legality check is needed if we had trunc-of-lshr.
4746 if (HadTrunc && match(WidestShift, m_LShr(m_Value(), m_Value()))) {
4747 auto CanFold = [NewShAmt, WidestBitWidth, NarrowestShift, SQ,
4748 WidestShift]() {
4749 // It isn't obvious whether it's worth it to analyze non-constants here.
4750 // Also, let's basically give up on non-splat cases, pessimizing vectors.
4751 // If *any* of these preconditions matches we can perform the fold.
4752 Constant *NewShAmtSplat = NewShAmt->getType()->isVectorTy()
4753 ? NewShAmt->getSplatValue()
4754 : NewShAmt;
4755 // If it's edge-case shift (by 0 or by WidestBitWidth-1) we can fold.
4756 if (NewShAmtSplat &&
4757 (NewShAmtSplat->isNullValue() ||
4758 NewShAmtSplat->getUniqueInteger() == WidestBitWidth - 1))
4759 return true;
4760 // We consider *min* leading zeros so a single outlier
4761 // blocks the transform as opposed to allowing it.
4762 if (auto *C = dyn_cast<Constant>(NarrowestShift->getOperand(0))) {
4763 KnownBits Known = computeKnownBits(C, SQ.DL);
4764 unsigned MinLeadZero = Known.countMinLeadingZeros();
4765 // If the value being shifted has at most lowest bit set we can fold.
4766 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero;
4767 if (MaxActiveBits <= 1)
4768 return true;
4769 // Precondition: NewShAmt u<= countLeadingZeros(C)
4770 if (NewShAmtSplat && NewShAmtSplat->getUniqueInteger().ule(MinLeadZero))
4771 return true;
4772 }
4773 if (auto *C = dyn_cast<Constant>(WidestShift->getOperand(0))) {
4774 KnownBits Known = computeKnownBits(C, SQ.DL);
4775 unsigned MinLeadZero = Known.countMinLeadingZeros();
4776 // If the value being shifted has at most lowest bit set we can fold.
4777 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero;
4778 if (MaxActiveBits <= 1)
4779 return true;
4780 // Precondition: ((WidestBitWidth-1)-NewShAmt) u<= countLeadingZeros(C)
4781 if (NewShAmtSplat) {
4782 APInt AdjNewShAmt =
4783 (WidestBitWidth - 1) - NewShAmtSplat->getUniqueInteger();
4784 if (AdjNewShAmt.ule(MinLeadZero))
4785 return true;
4786 }
4787 }
4788 return false; // Can't tell if it's ok.
4789 };
4790 if (!CanFold())
4791 return nullptr;
4792 }
4793
4794 // All good, we can do this fold.
4795 X = Builder.CreateZExt(X, WidestTy);
4796 Y = Builder.CreateZExt(Y, WidestTy);
4797 // The shift is the same that was for X.
4798 Value *T0 = XShiftOpcode == Instruction::BinaryOps::LShr
4799 ? Builder.CreateLShr(X, NewShAmt)
4800 : Builder.CreateShl(X, NewShAmt);
4801 Value *T1 = Builder.CreateAnd(T0, Y);
4802 return Builder.CreateICmp(I.getPredicate(), T1,
4803 Constant::getNullValue(WidestTy));
4804}
4805
4806/// Fold
4807/// (-1 u/ x) u< y
4808/// ((x * y) ?/ x) != y
4809/// to
4810/// @llvm.?mul.with.overflow(x, y) plus extraction of overflow bit
4811/// Note that the comparison is commutative, while inverted (u>=, ==) predicate
4812/// will mean that we are looking for the opposite answer.
4814 CmpPredicate Pred;
4815 Value *X, *Y;
4817 Instruction *Div;
4818 bool NeedNegation;
4819 // Look for: (-1 u/ x) u</u>= y
4820 if (!I.isEquality() &&
4821 match(&I, m_c_ICmp(Pred,
4823 m_Instruction(Div)),
4824 m_Value(Y)))) {
4825 Mul = nullptr;
4826
4827 // Are we checking that overflow does not happen, or does happen?
4828 switch (Pred) {
4830 NeedNegation = false;
4831 break; // OK
4833 NeedNegation = true;
4834 break; // OK
4835 default:
4836 return nullptr; // Wrong predicate.
4837 }
4838 } else // Look for: ((x * y) / x) !=/== y
4839 if (I.isEquality() &&
4840 match(&I,
4841 m_c_ICmp(Pred, m_Value(Y),
4844 m_Value(X)),
4846 m_Deferred(X))),
4847 m_Instruction(Div))))) {
4848 NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ;
4849 } else
4850 return nullptr;
4851
4853 // If the pattern included (x * y), we'll want to insert new instructions
4854 // right before that original multiplication so that we can replace it.
4855 bool MulHadOtherUses = Mul && !Mul->hasOneUse();
4856 if (MulHadOtherUses)
4858
4860 Div->getOpcode() == Instruction::UDiv ? Intrinsic::umul_with_overflow
4861 : Intrinsic::smul_with_overflow,
4862 X->getType(), {X, Y}, /*FMFSource=*/nullptr, "mul");
4863
4864 // If the multiplication was used elsewhere, to ensure that we don't leave
4865 // "duplicate" instructions, replace uses of that original multiplication
4866 // with the multiplication result from the with.overflow intrinsic.
4867 if (MulHadOtherUses)
4868 replaceInstUsesWith(*Mul, Builder.CreateExtractValue(Call, 0, "mul.val"));
4869
4870 Value *Res = Builder.CreateExtractValue(Call, 1, "mul.ov");
4871 if (NeedNegation) // This technically increases instruction count.
4872 Res = Builder.CreateNot(Res, "mul.not.ov");
4873
4874 // If we replaced the mul, erase it. Do this after all uses of Builder,
4875 // as the mul is used as insertion point.
4876 if (MulHadOtherUses)
4878
4879 return Res;
4880}
4881
4883 InstCombiner::BuilderTy &Builder) {
4884 CmpPredicate Pred;
4885 Value *X;
4886 if (match(&I, m_c_ICmp(Pred, m_NSWNeg(m_Value(X)), m_Deferred(X)))) {
4887
4888 if (ICmpInst::isSigned(Pred))
4889 Pred = ICmpInst::getSwappedPredicate(Pred);
4890 else if (ICmpInst::isUnsigned(Pred))
4891 Pred = ICmpInst::getSignedPredicate(Pred);
4892 // else for equality-comparisons just keep the predicate.
4893
4894 return ICmpInst::Create(Instruction::ICmp, Pred, X,
4895 Constant::getNullValue(X->getType()), I.getName());
4896 }
4897
4898 // A value is not equal to its negation unless that value is 0 or
4899 // MinSignedValue, ie: a != -a --> (a & MaxSignedVal) != 0
4900 if (match(&I, m_c_ICmp(Pred, m_OneUse(m_Neg(m_Value(X))), m_Deferred(X))) &&
4901 ICmpInst::isEquality(Pred)) {
4902 Type *Ty = X->getType();
4904 Constant *MaxSignedVal =
4905 ConstantInt::get(Ty, APInt::getSignedMaxValue(BitWidth));
4906 Value *And = Builder.CreateAnd(X, MaxSignedVal);
4907 Constant *Zero = Constant::getNullValue(Ty);
4908 return CmpInst::Create(Instruction::ICmp, Pred, And, Zero);
4909 }
4910
4911 return nullptr;
4912}
4913
4915 InstCombinerImpl &IC) {
4916 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *A;
4917 // Normalize and operand as operand 0.
4918 CmpInst::Predicate Pred = I.getPredicate();
4919 if (match(Op1, m_c_And(m_Specific(Op0), m_Value()))) {
4920 std::swap(Op0, Op1);
4921 Pred = ICmpInst::getSwappedPredicate(Pred);
4922 }
4923
4924 if (!match(Op0, m_c_And(m_Specific(Op1), m_Value(A))))
4925 return nullptr;
4926
4927 // (icmp (X & Y) u< X --> (X & Y) != X
4928 if (Pred == ICmpInst::ICMP_ULT)
4929 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
4930
4931 // (icmp (X & Y) u>= X --> (X & Y) == X
4932 if (Pred == ICmpInst::ICMP_UGE)
4933 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
4934
4935 if (ICmpInst::isEquality(Pred) && Op0->hasOneUse()) {
4936 // icmp (X & Y) eq/ne Y --> (X | ~Y) eq/ne -1 if Y is freely invertible and
4937 // Y is non-constant. If Y is constant the `X & C == C` form is preferable
4938 // so don't do this fold.
4939 if (!match(Op1, m_ImmConstant()))
4940 if (auto *NotOp1 =
4941 IC.getFreelyInverted(Op1, !Op1->hasNUsesOrMore(3), &IC.Builder))
4942 return new ICmpInst(Pred, IC.Builder.CreateOr(A, NotOp1),
4943 Constant::getAllOnesValue(Op1->getType()));
4944 // icmp (X & Y) eq/ne Y --> (~X & Y) eq/ne 0 if X is freely invertible.
4945 if (auto *NotA = IC.getFreelyInverted(A, A->hasOneUse(), &IC.Builder))
4946 return new ICmpInst(Pred, IC.Builder.CreateAnd(Op1, NotA),
4947 Constant::getNullValue(Op1->getType()));
4948 }
4949
4950 if (!ICmpInst::isSigned(Pred))
4951 return nullptr;
4952
4953 KnownBits KnownY = IC.computeKnownBits(A, /*Depth=*/0, &I);
4954 // (X & NegY) spred X --> (X & NegY) upred X
4955 if (KnownY.isNegative())
4956 return new ICmpInst(ICmpInst::getUnsignedPredicate(Pred), Op0, Op1);
4957
4958 if (Pred != ICmpInst::ICMP_SLE && Pred != ICmpInst::ICMP_SGT)
4959 return nullptr;
4960
4961 if (KnownY.isNonNegative())
4962 // (X & PosY) s<= X --> X s>= 0
4963 // (X & PosY) s> X --> X s< 0
4964 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
4965 Constant::getNullValue(Op1->getType()));
4966
4968 // (NegX & Y) s<= NegX --> Y s< 0
4969 // (NegX & Y) s> NegX --> Y s>= 0
4971 Constant::getNullValue(A->getType()));
4972
4973 return nullptr;
4974}
4975
4977 InstCombinerImpl &IC) {
4978 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *A;
4979
4980 // Normalize or operand as operand 0.
4981 CmpInst::Predicate Pred = I.getPredicate();
4982 if (match(Op1, m_c_Or(m_Specific(Op0), m_Value(A)))) {
4983 std::swap(Op0, Op1);
4984 Pred = ICmpInst::getSwappedPredicate(Pred);
4985 } else if (!match(Op0, m_c_Or(m_Specific(Op1), m_Value(A)))) {
4986 return nullptr;
4987 }
4988
4989 // icmp (X | Y) u<= X --> (X | Y) == X
4990 if (Pred == ICmpInst::ICMP_ULE)
4991 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
4992
4993 // icmp (X | Y) u> X --> (X | Y) != X
4994 if (Pred == ICmpInst::ICMP_UGT)
4995 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
4996
4997 if (ICmpInst::isEquality(Pred) && Op0->hasOneUse()) {
4998 // icmp (X | Y) eq/ne Y --> (X & ~Y) eq/ne 0 if Y is freely invertible
4999 if (Value *NotOp1 =
5000 IC.getFreelyInverted(Op1, !Op1->hasNUsesOrMore(3), &IC.Builder))
5001 return new ICmpInst(Pred, IC.Builder.CreateAnd(A, NotOp1),
5002 Constant::getNullValue(Op1->getType()));
5003 // icmp (X | Y) eq/ne Y --> (~X | Y) eq/ne -1 if X is freely invertible.
5004 if (Value *NotA = IC.getFreelyInverted(A, A->hasOneUse(), &IC.Builder))
5005 return new ICmpInst(Pred, IC.Builder.CreateOr(Op1, NotA),
5006 Constant::getAllOnesValue(Op1->getType()));
5007 }
5008 return nullptr;
5009}
5010
5012 InstCombinerImpl &IC) {
5013 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *A;
5014 // Normalize xor operand as operand 0.
5015 CmpInst::Predicate Pred = I.getPredicate();
5016 if (match(Op1, m_c_Xor(m_Specific(Op0), m_Value()))) {
5017 std::swap(Op0, Op1);
5018 Pred = ICmpInst::getSwappedPredicate(Pred);
5019 }
5020 if (!match(Op0, m_c_Xor(m_Specific(Op1), m_Value(A))))
5021 return nullptr;
5022
5023 // icmp (X ^ Y_NonZero) u>= X --> icmp (X ^ Y_NonZero) u> X
5024 // icmp (X ^ Y_NonZero) u<= X --> icmp (X ^ Y_NonZero) u< X
5025 // icmp (X ^ Y_NonZero) s>= X --> icmp (X ^ Y_NonZero) s> X
5026 // icmp (X ^ Y_NonZero) s<= X --> icmp (X ^ Y_NonZero) s< X
5028 if (PredOut != Pred && isKnownNonZero(A, Q))
5029 return new ICmpInst(PredOut, Op0, Op1);
5030
5031 return nullptr;
5032}
5033
5034/// Try to fold icmp (binop), X or icmp X, (binop).
5035/// TODO: A large part of this logic is duplicated in InstSimplify's
5036/// simplifyICmpWithBinOp(). We should be able to share that and avoid the code
5037/// duplication.
5039 const SimplifyQuery &SQ) {
5041 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
5042
5043 // Special logic for binary operators.
5044 BinaryOperator *BO0 = dyn_cast<BinaryOperator>(Op0);
5045 BinaryOperator *BO1 = dyn_cast<BinaryOperator>(Op1);
5046 if (!BO0 && !BO1)
5047 return nullptr;
5048
5049 if (Instruction *NewICmp = foldICmpXNegX(I, Builder))
5050 return NewICmp;
5051
5052 const CmpInst::Predicate Pred = I.getPredicate();
5053 Value *X;
5054
5055 // Convert add-with-unsigned-overflow comparisons into a 'not' with compare.
5056 // (Op1 + X) u</u>= Op1 --> ~Op1 u</u>= X
5057 if (match(Op0, m_OneUse(m_c_Add(m_Specific(Op1), m_Value(X)))) &&
5058 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
5059 return new ICmpInst(Pred, Builder.CreateNot(Op1), X);
5060 // Op0 u>/u<= (Op0 + X) --> X u>/u<= ~Op0
5061 if (match(Op1, m_OneUse(m_c_Add(m_Specific(Op0), m_Value(X)))) &&
5062 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE))
5063 return new ICmpInst(Pred, X, Builder.CreateNot(Op0));
5064
5065 {
5066 // (Op1 + X) + C u</u>= Op1 --> ~C - X u</u>= Op1
5067 Constant *C;
5068 if (match(Op0, m_OneUse(m_Add(m_c_Add(m_Specific(Op1), m_Value(X)),
5069 m_ImmConstant(C)))) &&
5070 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) {
5072 return new ICmpInst(Pred, Builder.CreateSub(C2, X), Op1);
5073 }
5074 // Op0 u>/u<= (Op0 + X) + C --> Op0 u>/u<= ~C - X
5075 if (match(Op1, m_OneUse(m_Add(m_c_Add(m_Specific(Op0), m_Value(X)),
5076 m_ImmConstant(C)))) &&
5077 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) {
5079 return new ICmpInst(Pred, Op0, Builder.CreateSub(C2, X));
5080 }
5081 }
5082
5083 // (icmp eq/ne (X, -P2), INT_MIN)
5084 // -> (icmp slt/sge X, INT_MIN + P2)
5085 if (ICmpInst::isEquality(Pred) && BO0 &&
5086 match(I.getOperand(1), m_SignMask()) &&
5088 // Will Constant fold.
5089 Value *NewC = Builder.CreateSub(I.getOperand(1), BO0->getOperand(1));
5090 return new ICmpInst(Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_SLT
5092 BO0->getOperand(0), NewC);
5093 }
5094
5095 {
5096 // Similar to above: an unsigned overflow comparison may use offset + mask:
5097 // ((Op1 + C) & C) u< Op1 --> Op1 != 0
5098 // ((Op1 + C) & C) u>= Op1 --> Op1 == 0
5099 // Op0 u> ((Op0 + C) & C) --> Op0 != 0
5100 // Op0 u<= ((Op0 + C) & C) --> Op0 == 0
5101 BinaryOperator *BO;
5102 const APInt *C;
5103 if ((Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) &&
5104 match(Op0, m_And(m_BinOp(BO), m_LowBitMask(C))) &&
5106 CmpInst::Predicate NewPred =
5108 Constant *Zero = ConstantInt::getNullValue(Op1->getType());
5109 return new ICmpInst(NewPred, Op1, Zero);
5110 }
5111
5112 if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) &&
5113 match(Op1, m_And(m_BinOp(BO), m_LowBitMask(C))) &&
5115 CmpInst::Predicate NewPred =
5117 Constant *Zero = ConstantInt::getNullValue(Op1->getType());
5118 return new ICmpInst(NewPred, Op0, Zero);
5119 }
5120 }
5121
5122 bool NoOp0WrapProblem = false, NoOp1WrapProblem = false;
5123 bool Op0HasNUW = false, Op1HasNUW = false;
5124 bool Op0HasNSW = false, Op1HasNSW = false;
5125 // Analyze the case when either Op0 or Op1 is an add instruction.
5126 // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null).
5127 auto hasNoWrapProblem = [](const BinaryOperator &BO, CmpInst::Predicate Pred,
5128 bool &HasNSW, bool &HasNUW) -> bool {
5129 if (isa<OverflowingBinaryOperator>(BO)) {
5130 HasNUW = BO.hasNoUnsignedWrap();
5131 HasNSW = BO.hasNoSignedWrap();
5132 return ICmpInst::isEquality(Pred) ||
5133 (CmpInst::isUnsigned(Pred) && HasNUW) ||
5134 (CmpInst::isSigned(Pred) && HasNSW);
5135 } else if (BO.getOpcode() == Instruction::Or) {
5136 HasNUW = true;
5137 HasNSW = true;
5138 return true;
5139 } else {
5140 return false;
5141 }
5142 };
5143 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
5144
5145 if (BO0) {
5146 match(BO0, m_AddLike(m_Value(A), m_Value(B)));
5147 NoOp0WrapProblem = hasNoWrapProblem(*BO0, Pred, Op0HasNSW, Op0HasNUW);
5148 }
5149 if (BO1) {
5150 match(BO1, m_AddLike(m_Value(C), m_Value(D)));
5151 NoOp1WrapProblem = hasNoWrapProblem(*BO1, Pred, Op1HasNSW, Op1HasNUW);
5152 }
5153
5154 // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow.
5155 // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow.
5156 if ((A == Op1 || B == Op1) && NoOp0WrapProblem)
5157 return new ICmpInst(Pred, A == Op1 ? B : A,
5158 Constant::getNullValue(Op1->getType()));
5159
5160 // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow.
5161 // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow.
5162 if ((C == Op0 || D == Op0) && NoOp1WrapProblem)
5163 return new ICmpInst(Pred, Constant::getNullValue(Op0->getType()),
5164 C == Op0 ? D : C);
5165
5166 // icmp (A+B), (A+D) -> icmp B, D for equalities or if there is no overflow.
5167 if (A && C && (A == C || A == D || B == C || B == D) && NoOp0WrapProblem &&
5168 NoOp1WrapProblem) {
5169 // Determine Y and Z in the form icmp (X+Y), (X+Z).
5170 Value *Y, *Z;
5171 if (A == C) {
5172 // C + B == C + D -> B == D
5173 Y = B;
5174 Z = D;
5175 } else if (A == D) {
5176 // D + B == C + D -> B == C
5177 Y = B;
5178 Z = C;
5179 } else if (B == C) {
5180 // A + C == C + D -> A == D
5181 Y = A;
5182 Z = D;
5183 } else {
5184 assert(B == D);
5185 // A + D == C + D -> A == C
5186 Y = A;
5187 Z = C;
5188 }
5189 return new ICmpInst(Pred, Y, Z);
5190 }
5191
5192 // icmp slt (A + -1), Op1 -> icmp sle A, Op1
5193 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT &&
5194 match(B, m_AllOnes()))
5195 return new ICmpInst(CmpInst::ICMP_SLE, A, Op1);
5196
5197 // icmp sge (A + -1), Op1 -> icmp sgt A, Op1
5198 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE &&
5199 match(B, m_AllOnes()))
5200 return new ICmpInst(CmpInst::ICMP_SGT, A, Op1);
5201
5202 // icmp sle (A + 1), Op1 -> icmp slt A, Op1
5203 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(B, m_One()))
5204 return new ICmpInst(CmpInst::ICMP_SLT, A, Op1);
5205
5206 // icmp sgt (A + 1), Op1 -> icmp sge A, Op1
5207 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(B, m_One()))
5208 return new ICmpInst(CmpInst::ICMP_SGE, A, Op1);
5209
5210 // icmp sgt Op0, (C + -1) -> icmp sge Op0, C
5211 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT &&
5212 match(D, m_AllOnes()))
5213 return new ICmpInst(CmpInst::ICMP_SGE, Op0, C);
5214
5215 // icmp sle Op0, (C + -1) -> icmp slt Op0, C
5216 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE &&
5217 match(D, m_AllOnes()))
5218 return new ICmpInst(CmpInst::ICMP_SLT, Op0, C);
5219
5220 // icmp sge Op0, (C + 1) -> icmp sgt Op0, C
5221 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(D, m_One()))
5222 return new ICmpInst(CmpInst::ICMP_SGT, Op0, C);
5223
5224 // icmp slt Op0, (C + 1) -> icmp sle Op0, C
5225 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(D, m_One()))
5226 return new ICmpInst(CmpInst::ICMP_SLE, Op0, C);
5227
5228 // TODO: The subtraction-related identities shown below also hold, but
5229 // canonicalization from (X -nuw 1) to (X + -1) means that the combinations
5230 // wouldn't happen even if they were implemented.
5231 //
5232 // icmp ult (A - 1), Op1 -> icmp ule A, Op1
5233 // icmp uge (A - 1), Op1 -> icmp ugt A, Op1
5234 // icmp ugt Op0, (C - 1) -> icmp uge Op0, C
5235 // icmp ule Op0, (C - 1) -> icmp ult Op0, C
5236
5237 // icmp ule (A + 1), Op0 -> icmp ult A, Op1
5238 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(B, m_One()))
5239 return new ICmpInst(CmpInst::ICMP_ULT, A, Op1);
5240
5241 // icmp ugt (A + 1), Op0 -> icmp uge A, Op1
5242 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(B, m_One()))
5243 return new ICmpInst(CmpInst::ICMP_UGE, A, Op1);
5244
5245 // icmp uge Op0, (C + 1) -> icmp ugt Op0, C
5246 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(D, m_One()))
5247 return new ICmpInst(CmpInst::ICMP_UGT, Op0, C);
5248
5249 // icmp ult Op0, (C + 1) -> icmp ule Op0, C
5250 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(D, m_One()))
5251 return new ICmpInst(CmpInst::ICMP_ULE, Op0, C);
5252
5253 // if C1 has greater magnitude than C2:
5254 // icmp (A + C1), (C + C2) -> icmp (A + C3), C
5255 // s.t. C3 = C1 - C2
5256 //
5257 // if C2 has greater magnitude than C1:
5258 // icmp (A + C1), (C + C2) -> icmp A, (C + C3)
5259 // s.t. C3 = C2 - C1
5260 if (A && C && NoOp0WrapProblem && NoOp1WrapProblem &&
5261 (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) {
5262 const APInt *AP1, *AP2;
5263 // TODO: Support non-uniform vectors.
5264 // TODO: Allow poison passthrough if B or D's element is poison.
5265 if (match(B, m_APIntAllowPoison(AP1)) &&
5266 match(D, m_APIntAllowPoison(AP2)) &&
5267 AP1->isNegative() == AP2->isNegative()) {
5268 APInt AP1Abs = AP1->abs();
5269 APInt AP2Abs = AP2->abs();
5270 if (AP1Abs.uge(AP2Abs)) {
5271 APInt Diff = *AP1 - *AP2;
5272 Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff);
5273 Value *NewAdd = Builder.CreateAdd(
5274 A, C3, "", Op0HasNUW && Diff.ule(*AP1), Op0HasNSW);
5275 return new ICmpInst(Pred, NewAdd, C);
5276 } else {
5277 APInt Diff = *AP2 - *AP1;
5278 Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff);
5279 Value *NewAdd = Builder.CreateAdd(
5280 C, C3, "", Op1HasNUW && Diff.ule(*AP2), Op1HasNSW);
5281 return new ICmpInst(Pred, A, NewAdd);
5282 }
5283 }
5284 Constant *Cst1, *Cst2;
5285 if (match(B, m_ImmConstant(Cst1)) && match(D, m_ImmConstant(Cst2)) &&
5286 ICmpInst::isEquality(Pred)) {
5287 Constant *Diff = ConstantExpr::getSub(Cst2, Cst1);
5288 Value *NewAdd = Builder.CreateAdd(C, Diff);
5289 return new ICmpInst(Pred, A, NewAdd);
5290 }
5291 }
5292
5293 // Analyze the case when either Op0 or Op1 is a sub instruction.
5294 // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null).
5295 A = nullptr;
5296 B = nullptr;
5297 C = nullptr;
5298 D = nullptr;
5299 if (BO0 && BO0->getOpcode() == Instruction::Sub) {
5300 A = BO0->getOperand(0);
5301 B = BO0->getOperand(1);
5302 }
5303 if (BO1 && BO1->getOpcode() == Instruction::Sub) {
5304 C = BO1->getOperand(0);
5305 D = BO1->getOperand(1);
5306 }
5307
5308 // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow.
5309 if (A == Op1 && NoOp0WrapProblem)
5310 return new ICmpInst(Pred, Constant::getNullValue(Op1->getType()), B);
5311 // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow.
5312 if (C == Op0 && NoOp1WrapProblem)
5313 return new ICmpInst(Pred, D, Constant::getNullValue(Op0->getType()));
5314
5315 // Convert sub-with-unsigned-overflow comparisons into a comparison of args.
5316 // (A - B) u>/u<= A --> B u>/u<= A
5317 if (A == Op1 && (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE))
5318 return new ICmpInst(Pred, B, A);
5319 // C u</u>= (C - D) --> C u</u>= D
5320 if (C == Op0 && (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
5321 return new ICmpInst(Pred, C, D);
5322 // (A - B) u>=/u< A --> B u>/u<= A iff B != 0
5323 if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
5324 isKnownNonZero(B, Q))
5326 // C u<=/u> (C - D) --> C u</u>= D iff B != 0
5327 if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
5328 isKnownNonZero(D, Q))
5330
5331 // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow.
5332 if (B && D && B == D && NoOp0WrapProblem && NoOp1WrapProblem)
5333 return new ICmpInst(Pred, A, C);
5334
5335 // icmp (A-B), (A-D) -> icmp D, B for equalities or if there is no overflow.
5336 if (A && C && A == C && NoOp0WrapProblem && NoOp1WrapProblem)
5337 return new ICmpInst(Pred, D, B);
5338
5339 // icmp (0-X) < cst --> x > -cst
5340 if (NoOp0WrapProblem && ICmpInst::isSigned(Pred)) {
5341 Value *X;
5342 if (match(BO0, m_Neg(m_Value(X))))
5343 if (Constant *RHSC = dyn_cast<Constant>(Op1))
5344 if (RHSC->isNotMinSignedValue())
5345 return new ICmpInst(I.getSwappedPredicate(), X,
5346 ConstantExpr::getNeg(RHSC));
5347 }
5348
5349 if (Instruction * R = foldICmpXorXX(I, Q, *this))
5350 return R;
5351 if (Instruction *R = foldICmpOrXX(I, Q, *this))
5352 return R;
5353
5354 {
5355 // Try to remove shared multiplier from comparison:
5356 // X * Z pred Y * Z
5357 Value *X, *Y, *Z;
5358 if ((match(Op0, m_Mul(m_Value(X), m_Value(Z))) &&
5359 match(Op1, m_c_Mul(m_Specific(Z), m_Value(Y)))) ||
5360 (match(Op0, m_Mul(m_Value(Z), m_Value(X))) &&
5361 match(Op1, m_c_Mul(m_Specific(Z), m_Value(Y))))) {
5362 if (ICmpInst::isSigned(Pred)) {
5363 if (Op0HasNSW && Op1HasNSW) {
5364 KnownBits ZKnown = computeKnownBits(Z, 0, &I);
5365 if (ZKnown.isStrictlyPositive())
5366 return new ICmpInst(Pred, X, Y);
5367 if (ZKnown.isNegative())
5368 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), X, Y);
5371 if (LessThan && match(LessThan, m_One()))
5372 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Z,
5373 Constant::getNullValue(Z->getType()));
5374 Value *GreaterThan = simplifyICmpInst(ICmpInst::ICMP_SGT, X, Y,
5376 if (GreaterThan && match(GreaterThan, m_One()))
5377 return new ICmpInst(Pred, Z, Constant::getNullValue(Z->getType()));
5378 }
5379 } else {
5380 bool NonZero;
5381 if (ICmpInst::isEquality(Pred)) {
5382 // If X != Y, fold (X *nw Z) eq/ne (Y *nw Z) -> Z eq/ne 0
5383 if (((Op0HasNSW && Op1HasNSW) || (Op0HasNUW && Op1HasNUW)) &&
5384 isKnownNonEqual(X, Y, DL, &AC, &I, &DT))
5385 return new ICmpInst(Pred, Z, Constant::getNullValue(Z->getType()));
5386
5387 KnownBits ZKnown = computeKnownBits(Z, 0, &I);
5388 // if Z % 2 != 0
5389 // X * Z eq/ne Y * Z -> X eq/ne Y
5390 if (ZKnown.countMaxTrailingZeros() == 0)
5391 return new ICmpInst(Pred, X, Y);
5392 NonZero = !ZKnown.One.isZero() || isKnownNonZero(Z, Q);
5393 // if Z != 0 and nsw(X * Z) and nsw(Y * Z)
5394 // X * Z eq/ne Y * Z -> X eq/ne Y
5395 if (NonZero && BO0 && BO1 && Op0HasNSW && Op1HasNSW)
5396 return new ICmpInst(Pred, X, Y);
5397 } else
5398 NonZero = isKnownNonZero(Z, Q);
5399
5400 // If Z != 0 and nuw(X * Z) and nuw(Y * Z)
5401 // X * Z u{lt/le/gt/ge}/eq/ne Y * Z -> X u{lt/le/gt/ge}/eq/ne Y
5402 if (NonZero && BO0 && BO1 && Op0HasNUW && Op1HasNUW)
5403 return new ICmpInst(Pred, X, Y);
5404 }
5405 }
5406 }
5407
5408 BinaryOperator *SRem = nullptr;
5409 // icmp (srem X, Y), Y
5410 if (BO0 && BO0->getOpcode() == Instruction::SRem && Op1 == BO0->getOperand(1))
5411 SRem = BO0;
5412 // icmp Y, (srem X, Y)
5413 else if (BO1 && BO1->getOpcode() == Instruction::SRem &&
5414 Op0 == BO1->getOperand(1))
5415 SRem = BO1;
5416 if (SRem) {
5417 // We don't check hasOneUse to avoid increasing register pressure because
5418 // the value we use is the same value this instruction was already using.
5419 switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(Pred) : Pred) {
5420 default:
5421 break;
5422 case ICmpInst::ICMP_EQ:
5423 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5424 case ICmpInst::ICMP_NE:
5425 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5426 case ICmpInst::ICMP_SGT:
5427 case ICmpInst::ICMP_SGE:
5428 return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(1),
5430 case ICmpInst::ICMP_SLT:
5431 case ICmpInst::ICMP_SLE:
5432 return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(1),
5434 }
5435 }
5436
5437 if (BO0 && BO1 && BO0->getOpcode() == BO1->getOpcode() &&
5438 (BO0->hasOneUse() || BO1->hasOneUse()) &&
5439 BO0->getOperand(1) == BO1->getOperand(1)) {
5440 switch (BO0->getOpcode()) {
5441 default:
5442 break;
5443 case Instruction::Add:
5444 case Instruction::Sub:
5445 case Instruction::Xor: {
5446 if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b
5447 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5448
5449 const APInt *C;
5450 if (match(BO0->getOperand(1), m_APInt(C))) {
5451 // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b
5452 if (C->isSignMask()) {
5453 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate();
5454 return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0));
5455 }
5456
5457 // icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b
5458 if (BO0->getOpcode() == Instruction::Xor && C->isMaxSignedValue()) {
5459 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate();
5460 NewPred = I.getSwappedPredicate(NewPred);
5461 return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0));
5462 }
5463 }
5464 break;
5465 }
5466 case Instruction::Mul: {
5467 if (!I.isEquality())
5468 break;
5469
5470 const APInt *C;
5471 if (match(BO0->getOperand(1), m_APInt(C)) && !C->isZero() &&
5472 !C->isOne()) {
5473 // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask)
5474 // Mask = -1 >> count-trailing-zeros(C).
5475 if (unsigned TZs = C->countr_zero()) {
5476 Constant *Mask = ConstantInt::get(
5477 BO0->getType(),
5478 APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs));
5479 Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask);
5480 Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask);
5481 return new ICmpInst(Pred, And1, And2);
5482 }
5483 }
5484 break;
5485 }
5486 case Instruction::UDiv:
5487 case Instruction::LShr:
5488 if (I.isSigned() || !BO0->isExact() || !BO1->isExact())
5489 break;
5490 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5491
5492 case Instruction::SDiv:
5493 if (!(I.isEquality() || match(BO0->getOperand(1), m_NonNegative())) ||
5494 !BO0->isExact() || !BO1->isExact())
5495 break;
5496 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5497
5498 case Instruction::AShr:
5499 if (!BO0->isExact() || !BO1->isExact())
5500 break;
5501 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5502
5503 case Instruction::Shl: {
5504 bool NUW = Op0HasNUW && Op1HasNUW;
5505 bool NSW = Op0HasNSW && Op1HasNSW;
5506 if (!NUW && !NSW)
5507 break;
5508 if (!NSW && I.isSigned())
5509 break;
5510 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5511 }
5512 }
5513 }
5514
5515 if (BO0) {
5516 // Transform A & (L - 1) `ult` L --> L != 0
5517 auto LSubOne = m_Add(m_Specific(Op1), m_AllOnes());
5518 auto BitwiseAnd = m_c_And(m_Value(), LSubOne);
5519
5520 if (match(BO0, BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) {
5521 auto *Zero = Constant::getNullValue(BO0->getType());
5522 return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero);
5523 }
5524 }
5525
5526 // For unsigned predicates / eq / ne:
5527 // icmp pred (x << 1), x --> icmp getSignedPredicate(pred) x, 0
5528 // icmp pred x, (x << 1) --> icmp getSignedPredicate(pred) 0, x
5529 if (!ICmpInst::isSigned(Pred)) {
5530 if (match(Op0, m_Shl(m_Specific(Op1), m_One())))
5531 return new ICmpInst(ICmpInst::getSignedPredicate(Pred), Op1,
5532 Constant::getNullValue(Op1->getType()));
5533 else if (match(Op1, m_Shl(m_Specific(Op0), m_One())))
5534 return new ICmpInst(ICmpInst::getSignedPredicate(Pred),
5535 Constant::getNullValue(Op0->getType()), Op0);
5536 }
5537
5539 return replaceInstUsesWith(I, V);
5540
5541 if (Instruction *R = foldICmpAndXX(I, Q, *this))
5542 return R;
5543
5545 return replaceInstUsesWith(I, V);
5546
5548 return replaceInstUsesWith(I, V);
5549
5550 return nullptr;
5551}
5552
5553/// Fold icmp Pred min|max(X, Y), Z.
5556 Value *Z, CmpPredicate Pred) {
5557 Value *X = MinMax->getLHS();
5558 Value *Y = MinMax->getRHS();
5559 if (ICmpInst::isSigned(Pred) && !MinMax->isSigned())
5560 return nullptr;
5561 if (ICmpInst::isUnsigned(Pred) && MinMax->isSigned()) {
5562 // Revert the transform signed pred -> unsigned pred
5563 // TODO: We can flip the signedness of predicate if both operands of icmp
5564 // are negative.
5568 } else
5569 return nullptr;
5570 }
5572 auto IsCondKnownTrue = [](Value *Val) -> std::optional<bool> {
5573 if (!Val)
5574 return std::nullopt;
5575 if (match(Val, m_One()))
5576 return true;
5577 if (match(Val, m_Zero()))
5578 return false;
5579 return std::nullopt;
5580 };
5581 auto CmpXZ = IsCondKnownTrue(simplifyICmpInst(Pred, X, Z, Q));
5582 auto CmpYZ = IsCondKnownTrue(simplifyICmpInst(Pred, Y, Z, Q));
5583 if (!CmpXZ.has_value() && !CmpYZ.has_value())
5584 return nullptr;
5585 if (!CmpXZ.has_value()) {
5586 std::swap(X, Y);
5587 std::swap(CmpXZ, CmpYZ);
5588 }
5589
5590 auto FoldIntoCmpYZ = [&]() -> Instruction * {
5591 if (CmpYZ.has_value())
5592 return replaceInstUsesWith(I, ConstantInt::getBool(I.getType(), *CmpYZ));
5593 return ICmpInst::Create(Instruction::ICmp, Pred, Y, Z);
5594 };
5595
5596 switch (Pred) {
5597 case ICmpInst::ICMP_EQ:
5598 case ICmpInst::ICMP_NE: {
5599 // If X == Z:
5600 // Expr Result
5601 // min(X, Y) == Z X <= Y
5602 // max(X, Y) == Z X >= Y
5603 // min(X, Y) != Z X > Y
5604 // max(X, Y) != Z X < Y
5605 if ((Pred == ICmpInst::ICMP_EQ) == *CmpXZ) {
5606 ICmpInst::Predicate NewPred =
5607 ICmpInst::getNonStrictPredicate(MinMax->getPredicate());
5608 if (Pred == ICmpInst::ICMP_NE)
5609 NewPred = ICmpInst::getInversePredicate(NewPred);
5610 return ICmpInst::Create(Instruction::ICmp, NewPred, X, Y);
5611 }
5612 // Otherwise (X != Z):
5613 ICmpInst::Predicate NewPred = MinMax->getPredicate();
5614 auto MinMaxCmpXZ = IsCondKnownTrue(simplifyICmpInst(NewPred, X, Z, Q));
5615 if (!MinMaxCmpXZ.has_value()) {
5616 std::swap(X, Y);
5617 std::swap(CmpXZ, CmpYZ);
5618 // Re-check pre-condition X != Z
5619 if (!CmpXZ.has_value() || (Pred == ICmpInst::ICMP_EQ) == *CmpXZ)
5620 break;
5621 MinMaxCmpXZ = IsCondKnownTrue(simplifyICmpInst(NewPred, X, Z, Q));
5622 }
5623 if (!MinMaxCmpXZ.has_value())
5624 break;
5625 if (*MinMaxCmpXZ) {
5626 // Expr Fact Result
5627 // min(X, Y) == Z X < Z false
5628 // max(X, Y) == Z X > Z false
5629 // min(X, Y) != Z X < Z true
5630 // max(X, Y) != Z X > Z true
5631 return replaceInstUsesWith(
5632 I, ConstantInt::getBool(I.getType(), Pred == ICmpInst::ICMP_NE));
5633 } else {
5634 // Expr Fact Result
5635 // min(X, Y) == Z X > Z Y == Z
5636 // max(X, Y) == Z X < Z Y == Z
5637 // min(X, Y) != Z X > Z Y != Z
5638 // max(X, Y) != Z X < Z Y != Z
5639 return FoldIntoCmpYZ();
5640 }
5641 break;
5642 }
5643 case ICmpInst::ICMP_SLT:
5644 case ICmpInst::ICMP_ULT:
5645 case ICmpInst::ICMP_SLE:
5646 case ICmpInst::ICMP_ULE:
5647 case ICmpInst::ICMP_SGT:
5648 case ICmpInst::ICMP_UGT:
5649 case ICmpInst::ICMP_SGE:
5650 case ICmpInst::ICMP_UGE: {
5651 bool IsSame = MinMax->getPredicate() == ICmpInst::getStrictPredicate(Pred);
5652 if (*CmpXZ) {
5653 if (IsSame) {
5654 // Expr Fact Result
5655 // min(X, Y) < Z X < Z true
5656 // min(X, Y) <= Z X <= Z true
5657 // max(X, Y) > Z X > Z true
5658 // max(X, Y) >= Z X >= Z true
5659 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5660 } else {
5661 // Expr Fact Result
5662 // max(X, Y) < Z X < Z Y < Z
5663 // max(X, Y) <= Z X <= Z Y <= Z
5664 // min(X, Y) > Z X > Z Y > Z
5665 // min(X, Y) >= Z X >= Z Y >= Z
5666 return FoldIntoCmpYZ();
5667 }
5668 } else {
5669 if (IsSame) {
5670 // Expr Fact Result
5671 // min(X, Y) < Z X >= Z Y < Z
5672 // min(X, Y) <= Z X > Z Y <= Z
5673 // max(X, Y) > Z X <= Z Y > Z
5674 // max(X, Y) >= Z X < Z Y >= Z
5675 return FoldIntoCmpYZ();
5676 } else {
5677 // Expr Fact Result
5678 // max(X, Y) < Z X >= Z false
5679 // max(X, Y) <= Z X > Z false
5680 // min(X, Y) > Z X <= Z false
5681 // min(X, Y) >= Z X < Z false
5682 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5683 }
5684 }
5685 break;
5686 }
5687 default:
5688 break;
5689 }
5690
5691 return nullptr;
5692}
5693
5694// Canonicalize checking for a power-of-2-or-zero value:
5696 InstCombiner::BuilderTy &Builder) {
5697 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
5698 const CmpInst::Predicate Pred = I.getPredicate();
5699 Value *A = nullptr;
5700 bool CheckIs;
5701 if (I.isEquality()) {
5702 // (A & (A-1)) == 0 --> ctpop(A) < 2 (two commuted variants)
5703 // ((A-1) & A) != 0 --> ctpop(A) > 1 (two commuted variants)
5704 if (!match(Op0, m_OneUse(m_c_And(m_Add(m_Value(A), m_AllOnes()),
5705 m_Deferred(A)))) ||
5706 !match(Op1, m_ZeroInt()))
5707 A = nullptr;
5708
5709 // (A & -A) == A --> ctpop(A) < 2 (four commuted variants)
5710 // (-A & A) != A --> ctpop(A) > 1 (four commuted variants)
5711 if (match(Op0, m_OneUse(m_c_And(m_Neg(m_Specific(Op1)), m_Specific(Op1)))))
5712 A = Op1;
5713 else if (match(Op1,
5715 A = Op0;
5716
5717 CheckIs = Pred == ICmpInst::ICMP_EQ;
5718 } else if (ICmpInst::isUnsigned(Pred)) {
5719 // (A ^ (A-1)) u>= A --> ctpop(A) < 2 (two commuted variants)
5720 // ((A-1) ^ A) u< A --> ctpop(A) > 1 (two commuted variants)
5721
5722 if ((Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
5724 m_Specific(Op1))))) {
5725 A = Op1;
5726 CheckIs = Pred == ICmpInst::ICMP_UGE;
5727 } else if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) &&
5729 m_Specific(Op0))))) {
5730 A = Op0;
5731 CheckIs = Pred == ICmpInst::ICMP_ULE;
5732 }
5733 }
5734
5735 if (A) {
5736 Type *Ty = A->getType();
5737 CallInst *CtPop = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, A);
5738 return CheckIs ? new ICmpInst(ICmpInst::ICMP_ULT, CtPop,
5739 ConstantInt::get(Ty, 2))
5740 : new ICmpInst(ICmpInst::ICMP_UGT, CtPop,
5741 ConstantInt::get(Ty, 1));
5742 }
5743
5744 return nullptr;
5745}
5746
5748 if (!I.isEquality())
5749 return nullptr;
5750
5751 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
5752 const CmpInst::Predicate Pred = I.getPredicate();
5753 Value *A, *B, *C, *D;
5754 if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) {
5755 if (A == Op1 || B == Op1) { // (A^B) == A -> B == 0
5756 Value *OtherVal = A == Op1 ? B : A;
5757 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType()));
5758 }
5759
5760 if (match(Op1, m_Xor(m_Value(C), m_Value(D)))) {
5761 // A^c1 == C^c2 --> A == C^(c1^c2)
5762 ConstantInt *C1, *C2;
5763 if (match(B, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2)) &&
5764 Op1->hasOneUse()) {
5765 Constant *NC = Builder.getInt(C1->getValue() ^ C2->getValue());
5767 return new ICmpInst(Pred, A, Xor);
5768 }
5769
5770 // A^B == A^D -> B == D
5771 if (A == C)
5772 return new ICmpInst(Pred, B, D);
5773 if (A == D)
5774 return new ICmpInst(Pred, B, C);
5775 if (B == C)
5776 return new ICmpInst(Pred, A, D);
5777 if (B == D)
5778 return new ICmpInst(Pred, A, C);
5779 }
5780 }
5781
5782 if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && (A == Op0 || B == Op0)) {
5783 // A == (A^B) -> B == 0
5784 Value *OtherVal = A == Op0 ? B : A;
5785 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType()));
5786 }
5787
5788 // (X&Z) == (Y&Z) -> (X^Y) & Z == 0
5789 if (match(Op0, m_And(m_Value(A), m_Value(B))) &&
5790 match(Op1, m_And(m_Value(C), m_Value(D)))) {
5791 Value *X = nullptr, *Y = nullptr, *Z = nullptr;
5792
5793 if (A == C) {
5794 X = B;
5795 Y = D;
5796 Z = A;
5797 } else if (A == D) {
5798 X = B;
5799 Y = C;
5800 Z = A;
5801 } else if (B == C) {
5802 X = A;
5803 Y = D;
5804 Z = B;
5805 } else if (B == D) {
5806 X = A;
5807 Y = C;
5808 Z = B;
5809 }
5810
5811 if (X) {
5812 // If X^Y is a negative power of two, then `icmp eq/ne (Z & NegP2), 0`
5813 // will fold to `icmp ult/uge Z, -NegP2` incurringb no additional
5814 // instructions.
5815 const APInt *C0, *C1;
5816 bool XorIsNegP2 = match(X, m_APInt(C0)) && match(Y, m_APInt(C1)) &&
5817 (*C0 ^ *C1).isNegatedPowerOf2();
5818
5819 // If either Op0/Op1 are both one use or X^Y will constant fold and one of
5820 // Op0/Op1 are one use, proceed. In those cases we are instruction neutral
5821 // but `icmp eq/ne A, 0` is easier to analyze than `icmp eq/ne A, B`.
5822 int UseCnt =
5823 int(Op0->hasOneUse()) + int(Op1->hasOneUse()) +
5824 (int(match(X, m_ImmConstant()) && match(Y, m_ImmConstant())));
5825 if (XorIsNegP2 || UseCnt >= 2) {
5826 // Build (X^Y) & Z
5827 Op1 = Builder.CreateXor(X, Y);
5828 Op1 = Builder.CreateAnd(Op1, Z);
5829 return new ICmpInst(Pred, Op1, Constant::getNullValue(Op1->getType()));
5830 }
5831 }
5832 }
5833
5834 {
5835 // Similar to above, but specialized for constant because invert is needed:
5836 // (X | C) == (Y | C) --> (X ^ Y) & ~C == 0
5837 Value *X, *Y;
5838 Constant *C;
5839 if (match(Op0, m_OneUse(m_Or(m_Value(X), m_Constant(C)))) &&
5840 match(Op1, m_OneUse(m_Or(m_Value(Y), m_Specific(C))))) {
5843 return new ICmpInst(Pred, And, Constant::getNullValue(And->getType()));
5844 }
5845 }
5846
5847 if (match(Op1, m_ZExt(m_Value(A))) &&
5848 (Op0->hasOneUse() || Op1->hasOneUse())) {
5849 // (B & (Pow2C-1)) == zext A --> A == trunc B
5850 // (B & (Pow2C-1)) != zext A --> A != trunc B
5851 const APInt *MaskC;
5852 if (match(Op0, m_And(m_Value(B), m_LowBitMask(MaskC))) &&
5853 MaskC->countr_one() == A->getType()->getScalarSizeInBits())
5854 return new ICmpInst(Pred, A, Builder.CreateTrunc(B, A->getType()));
5855 }
5856
5857 // (A >> C) == (B >> C) --> (A^B) u< (1 << C)
5858 // For lshr and ashr pairs.
5859 const APInt *AP1, *AP2;
5860 if ((match(Op0, m_OneUse(m_LShr(m_Value(A), m_APIntAllowPoison(AP1)))) &&
5861 match(Op1, m_OneUse(m_LShr(m_Value(B), m_APIntAllowPoison(AP2))))) ||
5862 (match(Op0, m_OneUse(m_AShr(m_Value(A), m_APIntAllowPoison(AP1)))) &&
5863 match(Op1, m_OneUse(m_AShr(m_Value(B), m_APIntAllowPoison(AP2)))))) {
5864 if (AP1 != AP2)
5865 return nullptr;
5866 unsigned TypeBits = AP1->getBitWidth();
5867 unsigned ShAmt = AP1->getLimitedValue(TypeBits);
5868 if (ShAmt < TypeBits && ShAmt != 0) {
5869 ICmpInst::Predicate NewPred =
5871 Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted");
5872 APInt CmpVal = APInt::getOneBitSet(TypeBits, ShAmt);
5873 return new ICmpInst(NewPred, Xor, ConstantInt::get(A->getType(), CmpVal));
5874 }
5875 }
5876
5877 // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0
5878 ConstantInt *Cst1;
5879 if (match(Op0, m_OneUse(m_Shl(m_Value(A), m_ConstantInt(Cst1)))) &&
5880 match(Op1, m_OneUse(m_Shl(m_Value(B), m_Specific(Cst1))))) {
5881 unsigned TypeBits = Cst1->getBitWidth();
5882 unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits);
5883 if (ShAmt < TypeBits && ShAmt != 0) {
5884 Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted");
5885 APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt);
5887 I.getName() + ".mask");
5888 return new ICmpInst(Pred, And, Constant::getNullValue(Cst1->getType()));
5889 }
5890 }
5891
5892 // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to
5893 // "icmp (and X, mask), cst"
5894 uint64_t ShAmt = 0;
5895 if (Op0->hasOneUse() &&
5896 match(Op0, m_Trunc(m_OneUse(m_LShr(m_Value(A), m_ConstantInt(ShAmt))))) &&
5897 match(Op1, m_ConstantInt(Cst1)) &&
5898 // Only do this when A has multiple uses. This is most important to do
5899 // when it exposes other optimizations.
5900 !A->hasOneUse()) {
5901 unsigned ASize = cast<IntegerType>(A->getType())->getPrimitiveSizeInBits();
5902
5903 if (ShAmt < ASize) {
5904 APInt MaskV =
5906 MaskV <<= ShAmt;
5907
5908 APInt CmpV = Cst1->getValue().zext(ASize);
5909 CmpV <<= ShAmt;
5910
5911 Value *Mask = Builder.CreateAnd(A, Builder.getInt(MaskV));
5912 return new ICmpInst(Pred, Mask, Builder.getInt(CmpV));
5913 }
5914 }
5915
5917 return ICmp;
5918
5919 // Match icmp eq (trunc (lshr A, BW), (ashr (trunc A), BW-1)), which checks the
5920 // top BW/2 + 1 bits are all the same. Create "A >=s INT_MIN && A <=s INT_MAX",
5921 // which we generate as "icmp ult (add A, 2^(BW-1)), 2^BW" to skip a few steps
5922 // of instcombine.
5923 unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
5924 if (match(Op0, m_AShr(m_Trunc(m_Value(A)), m_SpecificInt(BitWidth - 1))) &&
5926 A->getType()->getScalarSizeInBits() == BitWidth * 2 &&
5927 (I.getOperand(0)->hasOneUse() || I.getOperand(1)->hasOneUse())) {
5929 Value *Add = Builder.CreateAdd(A, ConstantInt::get(A->getType(), C));
5930 return new ICmpInst(Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT
5932 Add, ConstantInt::get(A->getType(), C.shl(1)));
5933 }
5934
5935 // Canonicalize:
5936 // Assume B_Pow2 != 0
5937 // 1. A & B_Pow2 != B_Pow2 -> A & B_Pow2 == 0
5938 // 2. A & B_Pow2 == B_Pow2 -> A & B_Pow2 != 0
5939 if (match(Op0, m_c_And(m_Specific(Op1), m_Value())) &&
5940 isKnownToBeAPowerOfTwo(Op1, /* OrZero */ false, 0, &I))
5941 return new ICmpInst(CmpInst::getInversePredicate(Pred), Op0,
5943
5944 if (match(Op1, m_c_And(m_Specific(Op0), m_Value())) &&
5945 isKnownToBeAPowerOfTwo(Op0, /* OrZero */ false, 0, &I))
5946 return new ICmpInst(CmpInst::getInversePredicate(Pred), Op1,
5947 ConstantInt::getNullValue(Op1->getType()));
5948
5949 // Canonicalize:
5950 // icmp eq/ne X, OneUse(rotate-right(X))
5951 // -> icmp eq/ne X, rotate-left(X)
5952 // We generally try to convert rotate-right -> rotate-left, this just
5953 // canonicalizes another case.
5954 if (match(&I, m_c_ICmp(m_Value(A),
5955 m_OneUse(m_Intrinsic<Intrinsic::fshr>(
5956 m_Deferred(A), m_Deferred(A), m_Value(B))))))
5957 return new ICmpInst(
5958 Pred, A,
5959 Builder.CreateIntrinsic(Op0->getType(), Intrinsic::fshl, {A, A, B}));
5960
5961 // Canonicalize:
5962 // icmp eq/ne OneUse(A ^ Cst), B --> icmp eq/ne (A ^ B), Cst
5963 Constant *Cst;
5966 return new ICmpInst(Pred, Builder.CreateXor(A, B), Cst);
5967
5968 {
5969 // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2)
5970 auto m_Matcher =
5973 m_Sub(m_Value(B), m_Deferred(A)));
5974 std::optional<bool> IsZero = std::nullopt;
5975 if (match(&I, m_c_ICmp(m_OneUse(m_c_And(m_Value(A), m_Matcher)),
5976 m_Deferred(A))))
5977 IsZero = false;
5978 // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0)
5979 else if (match(&I,
5980 m_ICmp(m_OneUse(m_c_And(m_Value(A), m_Matcher)), m_Zero())))
5981 IsZero = true;
5982
5983 if (IsZero && isKnownToBeAPowerOfTwo(A, /* OrZero */ true, /*Depth*/ 0, &I))
5984 // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2)
5985 // -> (icmp eq/ne (and X, P2), 0)
5986 // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0)
5987 // -> (icmp eq/ne (and X, P2), P2)
5988 return new ICmpInst(Pred, Builder.CreateAnd(B, A),
5989 *IsZero ? A
5990 : ConstantInt::getNullValue(A->getType()));
5991 }
5992
5993 return nullptr;
5994}
5995
5997 ICmpInst::Predicate Pred = ICmp.getPredicate();
5998 Value *Op0 = ICmp.getOperand(0), *Op1 = ICmp.getOperand(1);
5999
6000 // Try to canonicalize trunc + compare-to-constant into a mask + cmp.
6001 // The trunc masks high bits while the compare may effectively mask low bits.
6002 Value *X;
6003 const APInt *C;
6004 if (!match(Op0, m_OneUse(m_Trunc(m_Value(X)))) || !match(Op1, m_APInt(C)))
6005 return nullptr;
6006
6007 // This matches patterns corresponding to tests of the signbit as well as:
6008 // (trunc X) pred C2 --> (X & Mask) == C
6009 if (auto Res = decomposeBitTestICmp(Op0, Op1, Pred, /*WithTrunc=*/true,
6010 /*AllowNonZeroC=*/true)) {
6011 Value *And = Builder.CreateAnd(Res->X, Res->Mask);
6012 Constant *C = ConstantInt::get(Res->X->getType(), Res->C);
6013 return new ICmpInst(Res->Pred, And, C);
6014 }
6015
6016 unsigned SrcBits = X->getType()->getScalarSizeInBits();
6017 if (auto *II = dyn_cast<IntrinsicInst>(X)) {
6018 if (II->getIntrinsicID() == Intrinsic::cttz ||
6019 II->getIntrinsicID() == Intrinsic::ctlz) {
6020 unsigned MaxRet = SrcBits;
6021 // If the "is_zero_poison" argument is set, then we know at least
6022 // one bit is set in the input, so the result is always at least one
6023 // less than the full bitwidth of that input.
6024 if (match(II->getArgOperand(1), m_One()))
6025 MaxRet--;
6026
6027 // Make sure the destination is wide enough to hold the largest output of
6028 // the intrinsic.
6029 if (llvm::Log2_32(MaxRet) + 1 <= Op0->getType()->getScalarSizeInBits())
6030 if (Instruction *I =
6031 foldICmpIntrinsicWithConstant(ICmp, II, C->zext(SrcBits)))
6032 return I;
6033 }
6034 }
6035
6036 return nullptr;
6037}
6038
6040 assert(isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0");
6041 auto *CastOp0 = cast<CastInst>(ICmp.getOperand(0));
6042 Value *X;
6043 if (!match(CastOp0, m_ZExtOrSExt(m_Value(X))))
6044 return nullptr;
6045
6046 bool IsSignedExt = CastOp0->getOpcode() == Instruction::SExt;
6047 bool IsSignedCmp = ICmp.isSigned();
6048
6049 // icmp Pred (ext X), (ext Y)
6050 Value *Y;
6051 if (match(ICmp.getOperand(1), m_ZExtOrSExt(m_Value(Y)))) {
6052 bool IsZext0 = isa<ZExtInst>(ICmp.getOperand(0));
6053 bool IsZext1 = isa<ZExtInst>(ICmp.getOperand(1));
6054
6055 if (IsZext0 != IsZext1) {
6056 // If X and Y and both i1
6057 // (icmp eq/ne (zext X) (sext Y))
6058 // eq -> (icmp eq (or X, Y), 0)
6059 // ne -> (icmp ne (or X, Y), 0)
6060 if (ICmp.isEquality() && X->getType()->isIntOrIntVectorTy(1) &&
6061 Y->getType()->isIntOrIntVectorTy(1))
6062 return new ICmpInst(ICmp.getPredicate(), Builder.CreateOr(X, Y),
6063 Constant::getNullValue(X->getType()));
6064
6065 // If we have mismatched casts and zext has the nneg flag, we can
6066 // treat the "zext nneg" as "sext". Otherwise, we cannot fold and quit.
6067
6068 auto *NonNegInst0 = dyn_cast<PossiblyNonNegInst>(ICmp.getOperand(0));
6069 auto *NonNegInst1 = dyn_cast<PossiblyNonNegInst>(ICmp.getOperand(1));
6070
6071 bool IsNonNeg0 = NonNegInst0 && NonNegInst0->hasNonNeg();
6072 bool IsNonNeg1 = NonNegInst1 && NonNegInst1->hasNonNeg();
6073
6074 if ((IsZext0 && IsNonNeg0) || (IsZext1 && IsNonNeg1))
6075 IsSignedExt = true;
6076 else
6077 return nullptr;
6078 }
6079
6080 // Not an extension from the same type?
6081 Type *XTy = X->getType(), *YTy = Y->getType();
6082 if (XTy != YTy) {
6083 // One of the casts must have one use because we are creating a new cast.
6084 if (!ICmp.getOperand(0)->hasOneUse() && !ICmp.getOperand(1)->hasOneUse())
6085 return nullptr;
6086 // Extend the narrower operand to the type of the wider operand.
6087 CastInst::CastOps CastOpcode =
6088 IsSignedExt ? Instruction::SExt : Instruction::ZExt;
6089 if (XTy->getScalarSizeInBits() < YTy->getScalarSizeInBits())
6090 X = Builder.CreateCast(CastOpcode, X, YTy);
6091 else if (YTy->getScalarSizeInBits() < XTy->getScalarSizeInBits())
6092 Y = Builder.CreateCast(CastOpcode, Y, XTy);
6093 else
6094 return nullptr;
6095 }
6096
6097 // (zext X) == (zext Y) --> X == Y
6098 // (sext X) == (sext Y) --> X == Y
6099 if (ICmp.isEquality())
6100 return new ICmpInst(ICmp.getPredicate(), X, Y);
6101
6102 // A signed comparison of sign extended values simplifies into a
6103 // signed comparison.
6104 if (IsSignedCmp && IsSignedExt)
6105 return new ICmpInst(ICmp.getPredicate(), X, Y);
6106
6107 // The other three cases all fold into an unsigned comparison.
6108 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Y);
6109 }
6110
6111 // Below here, we are only folding a compare with constant.
6112 auto *C = dyn_cast<Constant>(ICmp.getOperand(1));
6113 if (!C)
6114 return nullptr;
6115
6116 // If a lossless truncate is possible...
6117 Type *SrcTy = CastOp0->getSrcTy();
6118 Constant *Res = getLosslessTrunc(C, SrcTy, CastOp0->getOpcode());
6119 if (Res) {
6120 if (ICmp.isEquality())
6121 return new ICmpInst(ICmp.getPredicate(), X, Res);
6122
6123 // A signed comparison of sign extended values simplifies into a
6124 // signed comparison.
6125 if (IsSignedExt && IsSignedCmp)
6126 return new ICmpInst(ICmp.getPredicate(), X, Res);
6127
6128 // The other three cases all fold into an unsigned comparison.
6129 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Res);
6130 }
6131
6132 // The re-extended constant changed, partly changed (in the case of a vector),
6133 // or could not be determined to be equal (in the case of a constant
6134 // expression), so the constant cannot be represented in the shorter type.
6135 // All the cases that fold to true or false will have already been handled
6136 // by simplifyICmpInst, so only deal with the tricky case.
6137 if (IsSignedCmp || !IsSignedExt || !isa<ConstantInt>(C))
6138 return nullptr;
6139
6140 // Is source op positive?
6141 // icmp ult (sext X), C --> icmp sgt X, -1
6142 if (ICmp.getPredicate() == ICmpInst::ICMP_ULT)
6144
6145 // Is source op negative?
6146 // icmp ugt (sext X), C --> icmp slt X, 0
6147 assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!");
6149}
6150
6151/// Handle icmp (cast x), (cast or constant).
6153 // If any operand of ICmp is a inttoptr roundtrip cast then remove it as
6154 // icmp compares only pointer's value.
6155 // icmp (inttoptr (ptrtoint p1)), p2 --> icmp p1, p2.
6156 Value *SimplifiedOp0 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(0));
6157 Value *SimplifiedOp1 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(1));
6158 if (SimplifiedOp0 || SimplifiedOp1)
6159 return new ICmpInst(ICmp.getPredicate(),
6160 SimplifiedOp0 ? SimplifiedOp0 : ICmp.getOperand(0),
6161 SimplifiedOp1 ? SimplifiedOp1 : ICmp.getOperand(1));
6162
6163 auto *CastOp0 = dyn_cast<CastInst>(ICmp.getOperand(0));
6164 if (!CastOp0)
6165 return nullptr;
6166 if (!isa<Constant>(ICmp.getOperand(1)) && !isa<CastInst>(ICmp.getOperand(1)))
6167 return nullptr;
6168
6169 Value *Op0Src = CastOp0->getOperand(0);
6170 Type *SrcTy = CastOp0->getSrcTy();
6171 Type *DestTy = CastOp0->getDestTy();
6172
6173 // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the
6174 // integer type is the same size as the pointer type.
6175 auto CompatibleSizes = [&](Type *PtrTy, Type *IntTy) {
6176 if (isa<VectorType>(PtrTy)) {
6177 PtrTy = cast<VectorType>(PtrTy)->getElementType();
6178 IntTy = cast<VectorType>(IntTy)->getElementType();
6179 }
6180 return DL.getPointerTypeSizeInBits(PtrTy) == IntTy->getIntegerBitWidth();
6181 };
6182 if (CastOp0->getOpcode() == Instruction::PtrToInt &&
6183 CompatibleSizes(SrcTy, DestTy)) {
6184 Value *NewOp1 = nullptr;
6185 if (auto *PtrToIntOp1 = dyn_cast<PtrToIntOperator>(ICmp.getOperand(1))) {
6186 Value *PtrSrc = PtrToIntOp1->getOperand(0);
6187 if (PtrSrc->getType() == Op0Src->getType())
6188 NewOp1 = PtrToIntOp1->getOperand(0);
6189 } else if (auto *RHSC = dyn_cast<Constant>(ICmp.getOperand(1))) {
6190 NewOp1 = ConstantExpr::getIntToPtr(RHSC, SrcTy);
6191 }
6192
6193 if (NewOp1)
6194 return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1);
6195 }
6196
6197 // Do the same in the other direction for icmp (inttoptr x), (inttoptr/c).
6198 if (CastOp0->getOpcode() == Instruction::IntToPtr &&
6199 CompatibleSizes(DestTy, SrcTy)) {
6200 Value *NewOp1 = nullptr;
6201 if (auto *IntToPtrOp1 = dyn_cast<IntToPtrInst>(ICmp.getOperand(1))) {
6202 Value *IntSrc = IntToPtrOp1->getOperand(0);
6203 if (IntSrc->getType() == Op0Src->getType())
6204 NewOp1 = IntToPtrOp1->getOperand(0);
6205 } else if (auto *RHSC = dyn_cast<Constant>(ICmp.getOperand(1))) {
6206 NewOp1 = ConstantFoldConstant(ConstantExpr::getPtrToInt(RHSC, SrcTy), DL);
6207 }
6208
6209 if (NewOp1)
6210 return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1);
6211 }
6212
6213 if (Instruction *R = foldICmpWithTrunc(ICmp))
6214 return R;
6215
6216 return foldICmpWithZextOrSext(ICmp);
6217}
6218
6219static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS, bool IsSigned) {
6220 switch (BinaryOp) {
6221 default:
6222 llvm_unreachable("Unsupported binary op");
6223 case Instruction::Add:
6224 case Instruction::Sub:
6225 return match(RHS, m_Zero());
6226 case Instruction::Mul:
6227 return !(RHS->getType()->isIntOrIntVectorTy(1) && IsSigned) &&
6228 match(RHS, m_One());
6229 }
6230}
6231
6234 bool IsSigned, Value *LHS, Value *RHS,
6235 Instruction *CxtI) const {
6236 switch (BinaryOp) {
6237 default:
6238 llvm_unreachable("Unsupported binary op");
6239 case Instruction::Add:
6240 if (IsSigned)
6241 return computeOverflowForSignedAdd(LHS, RHS, CxtI);
6242 else
6243 return computeOverflowForUnsignedAdd(LHS, RHS, CxtI);
6244 case Instruction::Sub:
6245 if (IsSigned)
6246 return computeOverflowForSignedSub(LHS, RHS, CxtI);
6247 else
6248 return computeOverflowForUnsignedSub(LHS, RHS, CxtI);
6249 case Instruction::Mul:
6250 if (IsSigned)
6251 return computeOverflowForSignedMul(LHS, RHS, CxtI);
6252 else
6253 return computeOverflowForUnsignedMul(LHS, RHS, CxtI);
6254 }
6255}
6256
6257bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp,
6258 bool IsSigned, Value *LHS,
6259 Value *RHS, Instruction &OrigI,
6260 Value *&Result,
6261 Constant *&Overflow) {
6262 if (OrigI.isCommutative() && isa<Constant>(LHS) && !isa<Constant>(RHS))
6263 std::swap(LHS, RHS);
6264
6265 // If the overflow check was an add followed by a compare, the insertion point
6266 // may be pointing to the compare. We want to insert the new instructions
6267 // before the add in case there are uses of the add between the add and the
6268 // compare.
6269 Builder.SetInsertPoint(&OrigI);
6270
6271 Type *OverflowTy = Type::getInt1Ty(LHS->getContext());
6272 if (auto *LHSTy = dyn_cast<VectorType>(LHS->getType()))
6273 OverflowTy = VectorType::get(OverflowTy, LHSTy->getElementCount());
6274
6275 if (isNeutralValue(BinaryOp, RHS, IsSigned)) {
6276 Result = LHS;
6277 Overflow = ConstantInt::getFalse(OverflowTy);
6278 return true;
6279 }
6280
6281 switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, &OrigI)) {
6283 return false;
6286 Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
6287 Result->takeName(&OrigI);
6288 Overflow = ConstantInt::getTrue(OverflowTy);
6289 return true;
6291 Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
6292 Result->takeName(&OrigI);
6293 Overflow = ConstantInt::getFalse(OverflowTy);
6294 if (auto *Inst = dyn_cast<Instruction>(Result)) {
6295 if (IsSigned)
6296 Inst->setHasNoSignedWrap();
6297 else
6298 Inst->setHasNoUnsignedWrap();
6299 }
6300 return true;
6301 }
6302
6303 llvm_unreachable("Unexpected overflow result");
6304}
6305
6306/// Recognize and process idiom involving test for multiplication
6307/// overflow.
6308///
6309/// The caller has matched a pattern of the form:
6310/// I = cmp u (mul(zext A, zext B), V
6311/// The function checks if this is a test for overflow and if so replaces
6312/// multiplication with call to 'mul.with.overflow' intrinsic.
6313///
6314/// \param I Compare instruction.
6315/// \param MulVal Result of 'mult' instruction. It is one of the arguments of
6316/// the compare instruction. Must be of integer type.
6317/// \param OtherVal The other argument of compare instruction.
6318/// \returns Instruction which must replace the compare instruction, NULL if no
6319/// replacement required.
6321 const APInt *OtherVal,
6322 InstCombinerImpl &IC) {
6323 // Don't bother doing this transformation for pointers, don't do it for
6324 // vectors.
6325 if (!isa<IntegerType>(MulVal->getType()))
6326 return nullptr;
6327
6328 auto *MulInstr = dyn_cast<Instruction>(MulVal);
6329 if (!MulInstr)
6330 return nullptr;
6331 assert(MulInstr->getOpcode() == Instruction::Mul);
6332
6333 auto *LHS = cast<ZExtInst>(MulInstr->getOperand(0)),
6334 *RHS = cast<ZExtInst>(MulInstr->getOperand(1));
6335 assert(LHS->getOpcode() == Instruction::ZExt);
6336 assert(RHS->getOpcode() == Instruction::ZExt);
6337 Value *A = LHS->getOperand(0), *B = RHS->getOperand(0);
6338
6339 // Calculate type and width of the result produced by mul.with.overflow.
6340 Type *TyA = A->getType(), *TyB = B->getType();
6341 unsigned WidthA = TyA->getPrimitiveSizeInBits(),
6342 WidthB = TyB->getPrimitiveSizeInBits();
6343 unsigned MulWidth;
6344 Type *MulType;
6345 if (WidthB > WidthA) {
6346 MulWidth = WidthB;
6347 MulType = TyB;
6348 } else {
6349 MulWidth = WidthA;
6350 MulType = TyA;
6351 }
6352
6353 // In order to replace the original mul with a narrower mul.with.overflow,
6354 // all uses must ignore upper bits of the product. The number of used low
6355 // bits must be not greater than the width of mul.with.overflow.
6356 if (MulVal->hasNUsesOrMore(2))
6357 for (User *U : MulVal->users()) {
6358 if (U == &I)
6359 continue;
6360 if (TruncInst *TI = dyn_cast<TruncInst>(U)) {
6361 // Check if truncation ignores bits above MulWidth.
6362 unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits();
6363 if (TruncWidth > MulWidth)
6364 return nullptr;
6365 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) {
6366 // Check if AND ignores bits above MulWidth.
6367 if (BO->getOpcode() != Instruction::And)
6368 return nullptr;
6369 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->getOperand(1))) {
6370 const APInt &CVal = CI->getValue();
6371 if (CVal.getBitWidth() - CVal.countl_zero() > MulWidth)
6372 return nullptr;
6373 } else {
6374 // In this case we could have the operand of the binary operation
6375 // being defined in another block, and performing the replacement
6376 // could break the dominance relation.
6377 return nullptr;
6378 }
6379 } else {
6380 // Other uses prohibit this transformation.
6381 return nullptr;
6382 }
6383 }
6384
6385 // Recognize patterns
6386 switch (I.getPredicate()) {
6387 case ICmpInst::ICMP_UGT: {
6388 // Recognize pattern:
6389 // mulval = mul(zext A, zext B)
6390 // cmp ugt mulval, max
6391 APInt MaxVal = APInt::getMaxValue(MulWidth);
6392 MaxVal = MaxVal.zext(OtherVal->getBitWidth());
6393 if (MaxVal.eq(*OtherVal))
6394 break; // Recognized
6395 return nullptr;
6396 }
6397
6398 case ICmpInst::ICMP_ULT: {
6399 // Recognize pattern:
6400 // mulval = mul(zext A, zext B)
6401 // cmp ule mulval, max + 1
6402 APInt MaxVal = APInt::getOneBitSet(OtherVal->getBitWidth(), MulWidth);
6403 if (MaxVal.eq(*OtherVal))
6404 break; // Recognized
6405 return nullptr;
6406 }
6407
6408 default:
6409 return nullptr;
6410 }
6411
6412 InstCombiner::BuilderTy &Builder = IC.Builder;
6413 Builder.SetInsertPoint(MulInstr);
6414
6415 // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B)
6416 Value *MulA = A, *MulB = B;
6417 if (WidthA < MulWidth)
6418 MulA = Builder.CreateZExt(A, MulType);
6419 if (WidthB < MulWidth)
6420 MulB = Builder.CreateZExt(B, MulType);
6421 CallInst *Call =
6422 Builder.CreateIntrinsic(Intrinsic::umul_with_overflow, MulType,
6423 {MulA, MulB}, /*FMFSource=*/nullptr, "umul");
6424 IC.addToWorklist(MulInstr);
6425
6426 // If there are uses of mul result other than the comparison, we know that
6427 // they are truncation or binary AND. Change them to use result of
6428 // mul.with.overflow and adjust properly mask/size.
6429 if (MulVal->hasNUsesOrMore(2)) {
6430 Value *Mul = Builder.CreateExtractValue(Call, 0, "umul.value");
6431 for (User *U : make_early_inc_range(MulVal->users())) {
6432 if (U == &I)
6433 continue;
6434 if (TruncInst *TI = dyn_cast<TruncInst>(U)) {
6435 if (TI->getType()->getPrimitiveSizeInBits() == MulWidth)
6436 IC.replaceInstUsesWith(*TI, Mul);
6437 else
6438 TI->setOperand(0, Mul);
6439 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) {
6440 assert(BO->getOpcode() == Instruction::And);
6441 // Replace (mul & mask) --> zext (mul.with.overflow & short_mask)
6442 ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1));
6443 APInt ShortMask = CI->getValue().trunc(MulWidth);
6444 Value *ShortAnd = Builder.CreateAnd(Mul, ShortMask);
6445 Value *Zext = Builder.CreateZExt(ShortAnd, BO->getType());
6446 IC.replaceInstUsesWith(*BO, Zext);
6447 } else {
6448 llvm_unreachable("Unexpected Binary operation");
6449 }
6450 IC.addToWorklist(cast<Instruction>(U));
6451 }
6452 }
6453
6454 // The original icmp gets replaced with the overflow value, maybe inverted
6455 // depending on predicate.
6456 if (I.getPredicate() == ICmpInst::ICMP_ULT) {
6457 Value *Res = Builder.CreateExtractValue(Call, 1);
6458 return BinaryOperator::CreateNot(Res);
6459 }
6460
6461 return ExtractValueInst::Create(Call, 1);
6462}
6463
6464/// When performing a comparison against a constant, it is possible that not all
6465/// the bits in the LHS are demanded. This helper method computes the mask that
6466/// IS demanded.
6468 const APInt *RHS;
6469 if (!match(I.getOperand(1), m_APInt(RHS)))
6471
6472 // If this is a normal comparison, it demands all bits. If it is a sign bit
6473 // comparison, it only demands the sign bit.
6474 bool UnusedBit;
6475 if (isSignBitCheck(I.getPredicate(), *RHS, UnusedBit))
6477
6478 switch (I.getPredicate()) {
6479 // For a UGT comparison, we don't care about any bits that
6480 // correspond to the trailing ones of the comparand. The value of these
6481 // bits doesn't impact the outcome of the comparison, because any value
6482 // greater than the RHS must differ in a bit higher than these due to carry.
6483 case ICmpInst::ICMP_UGT:
6484 return APInt::getBitsSetFrom(BitWidth, RHS->countr_one());
6485
6486 // Similarly, for a ULT comparison, we don't care about the trailing zeros.
6487 // Any value less than the RHS must differ in a higher bit because of carries.
6488 case ICmpInst::ICMP_ULT:
6489 return APInt::getBitsSetFrom(BitWidth, RHS->countr_zero());
6490
6491 default:
6493 }
6494}
6495
6496/// Check that one use is in the same block as the definition and all
6497/// other uses are in blocks dominated by a given block.
6498///
6499/// \param DI Definition
6500/// \param UI Use
6501/// \param DB Block that must dominate all uses of \p DI outside
6502/// the parent block
6503/// \return true when \p UI is the only use of \p DI in the parent block
6504/// and all other uses of \p DI are in blocks dominated by \p DB.
6505///
6507 const Instruction *UI,
6508 const BasicBlock *DB) const {
6509 assert(DI && UI && "Instruction not defined\n");
6510 // Ignore incomplete definitions.
6511 if (!DI->getParent())
6512 return false;
6513 // DI and UI must be in the same block.
6514 if (DI->getParent() != UI->getParent())
6515 return false;
6516 // Protect from self-referencing blocks.
6517 if (DI->getParent() == DB)
6518 return false;
6519 for (const User *U : DI->users()) {
6520 auto *Usr = cast<Instruction>(U);
6521 if (Usr != UI && !DT.dominates(DB, Usr->getParent()))
6522 return false;
6523 }
6524 return true;
6525}
6526
6527/// Return true when the instruction sequence within a block is select-cmp-br.
6528static bool isChainSelectCmpBranch(const SelectInst *SI) {
6529 const BasicBlock *BB = SI->getParent();
6530 if (!BB)
6531 return false;
6532 auto *BI = dyn_cast_or_null<BranchInst>(BB->getTerminator());
6533 if (!BI || BI->getNumSuccessors() != 2)
6534 return false;
6535 auto *IC = dyn_cast<ICmpInst>(BI->getCondition());
6536 if (!IC || (IC->getOperand(0) != SI && IC->getOperand(1) != SI))
6537 return false;
6538 return true;
6539}
6540
6541/// True when a select result is replaced by one of its operands
6542/// in select-icmp sequence. This will eventually result in the elimination
6543/// of the select.
6544///
6545/// \param SI Select instruction
6546/// \param Icmp Compare instruction
6547/// \param SIOpd Operand that replaces the select
6548///
6549/// Notes:
6550/// - The replacement is global and requires dominator information
6551/// - The caller is responsible for the actual replacement
6552///
6553/// Example:
6554///
6555/// entry:
6556/// %4 = select i1 %3, %C* %0, %C* null
6557/// %5 = icmp eq %C* %4, null
6558/// br i1 %5, label %9, label %7
6559/// ...
6560/// ; <label>:7 ; preds = %entry
6561/// %8 = getelementptr inbounds %C* %4, i64 0, i32 0
6562/// ...
6563///
6564/// can be transformed to
6565///
6566/// %5 = icmp eq %C* %0, null
6567/// %6 = select i1 %3, i1 %5, i1 true
6568/// br i1 %6, label %9, label %7
6569/// ...
6570/// ; <label>:7 ; preds = %entry
6571/// %8 = getelementptr inbounds %C* %0, i64 0, i32 0 // replace by %0!
6572///
6573/// Similar when the first operand of the select is a constant or/and
6574/// the compare is for not equal rather than equal.
6575///
6576/// NOTE: The function is only called when the select and compare constants
6577/// are equal, the optimization can work only for EQ predicates. This is not a
6578/// major restriction since a NE compare should be 'normalized' to an equal
6579/// compare, which usually happens in the combiner and test case
6580/// select-cmp-br.ll checks for it.
6582 const ICmpInst *Icmp,
6583 const unsigned SIOpd) {
6584 assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!");
6586 BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(1);
6587 // The check for the single predecessor is not the best that can be
6588 // done. But it protects efficiently against cases like when SI's
6589 // home block has two successors, Succ and Succ1, and Succ1 predecessor
6590 // of Succ. Then SI can't be replaced by SIOpd because the use that gets
6591 // replaced can be reached on either path. So the uniqueness check
6592 // guarantees that the path all uses of SI (outside SI's parent) are on
6593 // is disjoint from all other paths out of SI. But that information
6594 // is more expensive to compute, and the trade-off here is in favor
6595 // of compile-time. It should also be noticed that we check for a single
6596 // predecessor and not only uniqueness. This to handle the situation when
6597 // Succ and Succ1 points to the same basic block.
6598 if (Succ->getSinglePredecessor() && dominatesAllUses(SI, Icmp, Succ)) {
6599 NumSel++;
6600 SI->replaceUsesOutsideBlock(SI->getOperand(SIOpd), SI->getParent());
6601 return true;
6602 }
6603 }
6604 return false;
6605}
6606
6607/// Try to fold the comparison based on range information we can get by checking
6608/// whether bits are known to be zero or one in the inputs.
6610 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
6611 Type *Ty = Op0->getType();
6612 ICmpInst::Predicate Pred = I.getPredicate();
6613
6614 // Get scalar or pointer size.
6615 unsigned BitWidth = Ty->isIntOrIntVectorTy()
6616 ? Ty->getScalarSizeInBits()
6618
6619 if (!BitWidth)
6620 return nullptr;
6621
6622 KnownBits Op0Known(BitWidth);
6623 KnownBits Op1Known(BitWidth);
6624
6625 {
6626 // Don't use dominating conditions when folding icmp using known bits. This
6627 // may convert signed into unsigned predicates in ways that other passes
6628 // (especially IndVarSimplify) may not be able to reliably undo.
6631 Op0Known, /*Depth=*/0, Q))
6632 return &I;
6633
6635 /*Depth=*/0, Q))
6636 return &I;
6637 }
6638
6639 if (!isa<Constant>(Op0) && Op0Known.isConstant())
6640 return new ICmpInst(
6641 Pred, ConstantExpr::getIntegerValue(Ty, Op0Known.getConstant()), Op1);
6642 if (!isa<Constant>(Op1) && Op1Known.isConstant())
6643 return new ICmpInst(
6644 Pred, Op0, ConstantExpr::getIntegerValue(Ty, Op1Known.getConstant()));
6645
6646 if (std::optional<bool> Res = ICmpInst::compare(Op0Known, Op1Known, Pred))
6647 return replaceInstUsesWith(I, ConstantInt::getBool(I.getType(), *Res));
6648
6649 // Given the known and unknown bits, compute a range that the LHS could be
6650 // in. Compute the Min, Max and RHS values based on the known bits. For the
6651 // EQ and NE we use unsigned values.
6652 APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0);
6653 APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0);
6654 if (I.isSigned()) {
6655 Op0Min = Op0Known.getSignedMinValue();
6656 Op0Max = Op0Known.getSignedMaxValue();
6657 Op1Min = Op1Known.getSignedMinValue();
6658 Op1Max = Op1Known.getSignedMaxValue();
6659 } else {
6660 Op0Min = Op0Known.getMinValue();
6661 Op0Max = Op0Known.getMaxValue();
6662 Op1Min = Op1Known.getMinValue();
6663 Op1Max = Op1Known.getMaxValue();
6664 }
6665
6666 // Don't break up a clamp pattern -- (min(max X, Y), Z) -- by replacing a
6667 // min/max canonical compare with some other compare. That could lead to
6668 // conflict with select canonicalization and infinite looping.
6669 // FIXME: This constraint may go away if min/max intrinsics are canonical.
6670 auto isMinMaxCmp = [&](Instruction &Cmp) {
6671 if (!Cmp.hasOneUse())
6672 return false;
6673 Value *A, *B;
6674 SelectPatternFlavor SPF = matchSelectPattern(Cmp.user_back(), A, B).Flavor;
6676 return false;
6677 return match(Op0, m_MaxOrMin(m_Value(), m_Value())) ||
6678 match(Op1, m_MaxOrMin(m_Value(), m_Value()));
6679 };
6680 if (!isMinMaxCmp(I)) {
6681 switch (Pred) {
6682 default:
6683 break;
6684 case ICmpInst::ICMP_ULT: {
6685 if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B)
6686 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6687 const APInt *CmpC;
6688 if (match(Op1, m_APInt(CmpC))) {
6689 // A <u C -> A == C-1 if min(A)+1 == C
6690 if (*CmpC == Op0Min + 1)
6691 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6692 ConstantInt::get(Op1->getType(), *CmpC - 1));
6693 // X <u C --> X == 0, if the number of zero bits in the bottom of X
6694 // exceeds the log2 of C.
6695 if (Op0Known.countMinTrailingZeros() >= CmpC->ceilLogBase2())
6696 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6697 Constant::getNullValue(Op1->getType()));
6698 }
6699 break;
6700 }
6701 case ICmpInst::ICMP_UGT: {
6702 if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B)
6703 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6704 const APInt *CmpC;
6705 if (match(Op1, m_APInt(CmpC))) {
6706 // A >u C -> A == C+1 if max(a)-1 == C
6707 if (*CmpC == Op0Max - 1)
6708 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6709 ConstantInt::get(Op1->getType(), *CmpC + 1));
6710 // X >u C --> X != 0, if the number of zero bits in the bottom of X
6711 // exceeds the log2 of C.
6712 if (Op0Known.countMinTrailingZeros() >= CmpC->getActiveBits())
6713 return new ICmpInst(ICmpInst::ICMP_NE, Op0,
6714 Constant::getNullValue(Op1->getType()));
6715 }
6716 break;
6717 }
6718 case ICmpInst::ICMP_SLT: {
6719 if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B)
6720 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6721 const APInt *CmpC;
6722 if (match(Op1, m_APInt(CmpC))) {
6723 if (*CmpC == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C
6724 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6725 ConstantInt::get(Op1->getType(), *CmpC - 1));
6726 }
6727 break;
6728 }
6729 case ICmpInst::ICMP_SGT: {
6730 if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B)
6731 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6732 const APInt *CmpC;
6733 if (match(Op1, m_APInt(CmpC))) {
6734 if (*CmpC == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C
6735 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6736 ConstantInt::get(Op1->getType(), *CmpC + 1));
6737 }
6738 break;
6739 }
6740 }
6741 }
6742
6743 // Based on the range information we know about the LHS, see if we can
6744 // simplify this comparison. For example, (x&4) < 8 is always true.
6745 switch (Pred) {
6746 default:
6747 break;
6748 case ICmpInst::ICMP_EQ:
6749 case ICmpInst::ICMP_NE: {
6750 // If all bits are known zero except for one, then we know at most one bit
6751 // is set. If the comparison is against zero, then this is a check to see if
6752 // *that* bit is set.
6753 APInt Op0KnownZeroInverted = ~Op0Known.Zero;
6754 if (Op1Known.isZero()) {
6755 // If the LHS is an AND with the same constant, look through it.
6756 Value *LHS = nullptr;
6757 const APInt *LHSC;
6758 if (!match(Op0, m_And(m_Value(LHS), m_APInt(LHSC))) ||
6759 *LHSC != Op0KnownZeroInverted)
6760 LHS = Op0;
6761
6762 Value *X;
6763 const APInt *C1;
6764 if (match(LHS, m_Shl(m_Power2(C1), m_Value(X)))) {
6765 Type *XTy = X->getType();
6766 unsigned Log2C1 = C1->countr_zero();
6767 APInt C2 = Op0KnownZeroInverted;
6768 APInt C2Pow2 = (C2 & ~(*C1 - 1)) + *C1;
6769 if (C2Pow2.isPowerOf2()) {
6770 // iff (C1 is pow2) & ((C2 & ~(C1-1)) + C1) is pow2):
6771 // ((C1 << X) & C2) == 0 -> X >= (Log2(C2+C1) - Log2(C1))
6772 // ((C1 << X) & C2) != 0 -> X < (Log2(C2+C1) - Log2(C1))
6773 unsigned Log2C2 = C2Pow2.countr_zero();
6774 auto *CmpC = ConstantInt::get(XTy, Log2C2 - Log2C1);
6775 auto NewPred =
6777 return new ICmpInst(NewPred, X, CmpC);
6778 }
6779 }
6780 }
6781
6782 // Op0 eq C_Pow2 -> Op0 ne 0 if Op0 is known to be C_Pow2 or zero.
6783 if (Op1Known.isConstant() && Op1Known.getConstant().isPowerOf2() &&
6784 (Op0Known & Op1Known) == Op0Known)
6785 return new ICmpInst(CmpInst::getInversePredicate(Pred), Op0,
6786 ConstantInt::getNullValue(Op1->getType()));
6787 break;
6788 }
6789 case ICmpInst::ICMP_SGE:
6790 if (Op1Min == Op0Max) // A >=s B -> A == B if max(A) == min(B)
6791 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6792 break;
6793 case ICmpInst::ICMP_SLE:
6794 if (Op1Max == Op0Min) // A <=s B -> A == B if min(A) == max(B)
6795 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6796 break;
6797 case ICmpInst::ICMP_UGE:
6798 if (Op1Min == Op0Max) // A >=u B -> A == B if max(A) == min(B)
6799 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6800 break;
6801 case ICmpInst::ICMP_ULE:
6802 if (Op1Max == Op0Min) // A <=u B -> A == B if min(A) == max(B)
6803 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6804 break;
6805 }
6806
6807 // Turn a signed comparison into an unsigned one if both operands are known to
6808 // have the same sign. Set samesign if possible (except for equality
6809 // predicates).
6810 if ((I.isSigned() || (I.isUnsigned() && !I.hasSameSign())) &&
6811 ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) ||
6812 (Op0Known.One.isNegative() && Op1Known.One.isNegative()))) {
6813 I.setPredicate(I.getUnsignedPredicate());
6814 I.setSameSign();
6815 return &I;
6816 }
6817
6818 return nullptr;
6819}
6820
6821/// If one operand of an icmp is effectively a bool (value range of {0,1}),
6822/// then try to reduce patterns based on that limit.
6824 Value *X, *Y;
6825 CmpPredicate Pred;
6826
6827 // X must be 0 and bool must be true for "ULT":
6828 // X <u (zext i1 Y) --> (X == 0) & Y
6829 if (match(&I, m_c_ICmp(Pred, m_Value(X), m_OneUse(m_ZExt(m_Value(Y))))) &&
6830 Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULT)
6831 return BinaryOperator::CreateAnd(Builder.CreateIsNull(X), Y);
6832
6833 // X must be 0 or bool must be true for "ULE":
6834 // X <=u (sext i1 Y) --> (X == 0) | Y
6835 if (match(&I, m_c_ICmp(Pred, m_Value(X), m_OneUse(m_SExt(m_Value(Y))))) &&
6836 Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULE)
6837 return BinaryOperator::CreateOr(Builder.CreateIsNull(X), Y);
6838
6839 // icmp eq/ne X, (zext/sext (icmp eq/ne X, C))
6840 CmpPredicate Pred1, Pred2;
6841 const APInt *C;
6842 Instruction *ExtI;
6843 if (match(&I, m_c_ICmp(Pred1, m_Value(X),
6846 m_APInt(C)))))) &&
6847 ICmpInst::isEquality(Pred1) && ICmpInst::isEquality(Pred2)) {
6848 bool IsSExt = ExtI->getOpcode() == Instruction::SExt;
6849 bool HasOneUse = ExtI->hasOneUse() && ExtI->getOperand(0)->hasOneUse();
6850 auto CreateRangeCheck = [&] {
6851 Value *CmpV1 =
6852 Builder.CreateICmp(Pred1, X, Constant::getNullValue(X->getType()));
6853 Value *CmpV2 = Builder.CreateICmp(
6854 Pred1, X, ConstantInt::getSigned(X->getType(), IsSExt ? -1 : 1));
6856 Pred1 == ICmpInst::ICMP_EQ ? Instruction::Or : Instruction::And,
6857 CmpV1, CmpV2);
6858 };
6859 if (C->isZero()) {
6860 if (Pred2 == ICmpInst::ICMP_EQ) {
6861 // icmp eq X, (zext/sext (icmp eq X, 0)) --> false
6862 // icmp ne X, (zext/sext (icmp eq X, 0)) --> true
6863 return replaceInstUsesWith(
6864 I, ConstantInt::getBool(I.getType(), Pred1 == ICmpInst::ICMP_NE));
6865 } else if (!IsSExt || HasOneUse) {
6866 // icmp eq X, (zext (icmp ne X, 0)) --> X == 0 || X == 1
6867 // icmp ne X, (zext (icmp ne X, 0)) --> X != 0 && X != 1
6868 // icmp eq X, (sext (icmp ne X, 0)) --> X == 0 || X == -1
6869 // icmp ne X, (sext (icmp ne X, 0)) --> X != 0 && X == -1
6870 return CreateRangeCheck();
6871 }
6872 } else if (IsSExt ? C->isAllOnes() : C->isOne()) {
6873 if (Pred2 == ICmpInst::ICMP_NE) {
6874 // icmp eq X, (zext (icmp ne X, 1)) --> false
6875 // icmp ne X, (zext (icmp ne X, 1)) --> true
6876 // icmp eq X, (sext (icmp ne X, -1)) --> false
6877 // icmp ne X, (sext (icmp ne X, -1)) --> true
6878 return replaceInstUsesWith(
6879 I, ConstantInt::getBool(I.getType(), Pred1 == ICmpInst::ICMP_NE));
6880 } else if (!IsSExt || HasOneUse) {
6881 // icmp eq X, (zext (icmp eq X, 1)) --> X == 0 || X == 1
6882 // icmp ne X, (zext (icmp eq X, 1)) --> X != 0 && X != 1
6883 // icmp eq X, (sext (icmp eq X, -1)) --> X == 0 || X == -1
6884 // icmp ne X, (sext (icmp eq X, -1)) --> X != 0 && X == -1
6885 return CreateRangeCheck();
6886 }
6887 } else {
6888 // when C != 0 && C != 1:
6889 // icmp eq X, (zext (icmp eq X, C)) --> icmp eq X, 0
6890 // icmp eq X, (zext (icmp ne X, C)) --> icmp eq X, 1
6891 // icmp ne X, (zext (icmp eq X, C)) --> icmp ne X, 0
6892 // icmp ne X, (zext (icmp ne X, C)) --> icmp ne X, 1
6893 // when C != 0 && C != -1:
6894 // icmp eq X, (sext (icmp eq X, C)) --> icmp eq X, 0
6895 // icmp eq X, (sext (icmp ne X, C)) --> icmp eq X, -1
6896 // icmp ne X, (sext (icmp eq X, C)) --> icmp ne X, 0
6897 // icmp ne X, (sext (icmp ne X, C)) --> icmp ne X, -1
6898 return ICmpInst::Create(
6899 Instruction::ICmp, Pred1, X,
6900 ConstantInt::getSigned(X->getType(), Pred2 == ICmpInst::ICMP_NE
6901 ? (IsSExt ? -1 : 1)
6902 : 0));
6903 }
6904 }
6905
6906 return nullptr;
6907}
6908
6909std::optional<std::pair<CmpPredicate, Constant *>>
6911 Constant *C) {
6913 "Only for relational integer predicates.");
6914
6915 Type *Type = C->getType();
6916 bool IsSigned = ICmpInst::isSigned(Pred);
6917
6919 bool WillIncrement =
6920 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT;
6921
6922 // Check if the constant operand can be safely incremented/decremented
6923 // without overflowing/underflowing.
6924 auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) {
6925 return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned);
6926 };
6927
6928 Constant *SafeReplacementConstant = nullptr;
6929 if (auto *CI = dyn_cast<ConstantInt>(C)) {
6930 // Bail out if the constant can't be safely incremented/decremented.
6931 if (!ConstantIsOk(CI))
6932 return std::nullopt;
6933 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Type)) {
6934 unsigned NumElts = FVTy->getNumElements();
6935 for (unsigned i = 0; i != NumElts; ++i) {
6936 Constant *Elt = C->getAggregateElement(i);
6937 if (!Elt)
6938 return std::nullopt;
6939
6940 if (isa<UndefValue>(Elt))
6941 continue;
6942
6943 // Bail out if we can't determine if this constant is min/max or if we
6944 // know that this constant is min/max.
6945 auto *CI = dyn_cast<ConstantInt>(Elt);
6946 if (!CI || !ConstantIsOk(CI))
6947 return std::nullopt;
6948
6949 if (!SafeReplacementConstant)
6950 SafeReplacementConstant = CI;
6951 }
6952 } else if (isa<VectorType>(C->getType())) {
6953 // Handle scalable splat
6954 Value *SplatC = C->getSplatValue();
6955 auto *CI = dyn_cast_or_null<ConstantInt>(SplatC);
6956 // Bail out if the constant can't be safely incremented/decremented.
6957 if (!CI || !ConstantIsOk(CI))
6958 return std::nullopt;
6959 } else {
6960 // ConstantExpr?
6961 return std::nullopt;
6962 }
6963
6964 // It may not be safe to change a compare predicate in the presence of
6965 // undefined elements, so replace those elements with the first safe constant
6966 // that we found.
6967 // TODO: in case of poison, it is safe; let's replace undefs only.
6968 if (C->containsUndefOrPoisonElement()) {
6969 assert(SafeReplacementConstant && "Replacement constant not set");
6970 C = Constant::replaceUndefsWith(C, SafeReplacementConstant);
6971 }
6972
6974
6975 // Increment or decrement the constant.
6976 Constant *OneOrNegOne = ConstantInt::get(Type, WillIncrement ? 1 : -1, true);
6977 Constant *NewC = ConstantExpr::getAdd(C, OneOrNegOne);
6978
6979 return std::make_pair(NewPred, NewC);
6980}
6981
6982/// If we have an icmp le or icmp ge instruction with a constant operand, turn
6983/// it into the appropriate icmp lt or icmp gt instruction. This transform
6984/// allows them to be folded in visitICmpInst.
6986 ICmpInst::Predicate Pred = I.getPredicate();
6987 if (ICmpInst::isEquality(Pred) || !ICmpInst::isIntPredicate(Pred) ||
6989 return nullptr;
6990
6991 Value *Op0 = I.getOperand(0);
6992 Value *Op1 = I.getOperand(1);
6993 auto *Op1C = dyn_cast<Constant>(Op1);
6994 if (!Op1C)
6995 return nullptr;
6996
6997 auto FlippedStrictness =
6999 if (!FlippedStrictness)
7000 return nullptr;
7001
7002 return new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second);
7003}
7004
7005/// If we have a comparison with a non-canonical predicate, if we can update
7006/// all the users, invert the predicate and adjust all the users.
7008 // Is the predicate already canonical?
7009 CmpInst::Predicate Pred = I.getPredicate();
7011 return nullptr;
7012
7013 // Can all users be adjusted to predicate inversion?
7014 if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr))
7015 return nullptr;
7016
7017 // Ok, we can canonicalize comparison!
7018 // Let's first invert the comparison's predicate.
7019 I.setPredicate(CmpInst::getInversePredicate(Pred));
7020 I.setName(I.getName() + ".not");
7021
7022 // And, adapt users.
7024
7025 return &I;
7026}
7027
7028/// Integer compare with boolean values can always be turned into bitwise ops.
7030 InstCombiner::BuilderTy &Builder) {
7031 Value *A = I.getOperand(0), *B = I.getOperand(1);
7032 assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only");
7033
7034 // A boolean compared to true/false can be simplified to Op0/true/false in
7035 // 14 out of the 20 (10 predicates * 2 constants) possible combinations.
7036 // Cases not handled by InstSimplify are always 'not' of Op0.
7037 if (match(B, m_Zero())) {
7038 switch (I.getPredicate()) {
7039 case CmpInst::ICMP_EQ: // A == 0 -> !A
7040 case CmpInst::ICMP_ULE: // A <=u 0 -> !A
7041 case CmpInst::ICMP_SGE: // A >=s 0 -> !A
7043 default:
7044 llvm_unreachable("ICmp i1 X, C not simplified as expected.");
7045 }
7046 } else if (match(B, m_One())) {
7047 switch (I.getPredicate()) {
7048 case CmpInst::ICMP_NE: // A != 1 -> !A
7049 case CmpInst::ICMP_ULT: // A <u 1 -> !A
7050 case CmpInst::ICMP_SGT: // A >s -1 -> !A
7052 default:
7053 llvm_unreachable("ICmp i1 X, C not simplified as expected.");
7054 }
7055 }
7056
7057 switch (I.getPredicate()) {
7058 default:
7059 llvm_unreachable("Invalid icmp instruction!");
7060 case ICmpInst::ICMP_EQ:
7061 // icmp eq i1 A, B -> ~(A ^ B)
7062 return BinaryOperator::CreateNot(Builder.CreateXor(A, B));
7063
7064 case ICmpInst::ICMP_NE:
7065 // icmp ne i1 A, B -> A ^ B
7066 return BinaryOperator::CreateXor(A, B);
7067
7068 case ICmpInst::ICMP_UGT:
7069 // icmp ugt -> icmp ult
7070 std::swap(A, B);
7071 [[fallthrough]];
7072 case ICmpInst::ICMP_ULT:
7073 // icmp ult i1 A, B -> ~A & B
7074 return BinaryOperator::CreateAnd(Builder.CreateNot(A), B);
7075
7076 case ICmpInst::ICMP_SGT:
7077 // icmp sgt -> icmp slt
7078 std::swap(A, B);
7079 [[fallthrough]];
7080 case ICmpInst::ICMP_SLT:
7081 // icmp slt i1 A, B -> A & ~B
7082 return BinaryOperator::CreateAnd(Builder.CreateNot(B), A);
7083
7084 case ICmpInst::ICMP_UGE:
7085 // icmp uge -> icmp ule
7086 std::swap(A, B);
7087 [[fallthrough]];
7088 case ICmpInst::ICMP_ULE:
7089 // icmp ule i1 A, B -> ~A | B
7090 return BinaryOperator::CreateOr(Builder.CreateNot(A), B);
7091
7092 case ICmpInst::ICMP_SGE:
7093 // icmp sge -> icmp sle
7094 std::swap(A, B);
7095 [[fallthrough]];
7096 case ICmpInst::ICMP_SLE:
7097 // icmp sle i1 A, B -> A | ~B
7098 return BinaryOperator::CreateOr(Builder.CreateNot(B), A);
7099 }
7100}
7101
7102// Transform pattern like:
7103// (1 << Y) u<= X or ~(-1 << Y) u< X or ((1 << Y)+(-1)) u< X
7104// (1 << Y) u> X or ~(-1 << Y) u>= X or ((1 << Y)+(-1)) u>= X
7105// Into:
7106// (X l>> Y) != 0
7107// (X l>> Y) == 0
7109 InstCombiner::BuilderTy &Builder) {
7110 CmpPredicate Pred, NewPred;
7111 Value *X, *Y;
7112 if (match(&Cmp,
7113 m_c_ICmp(Pred, m_OneUse(m_Shl(m_One(), m_Value(Y))), m_Value(X)))) {
7114 switch (Pred) {
7115 case ICmpInst::ICMP_ULE:
7116 NewPred = ICmpInst::ICMP_NE;
7117 break;
7118 case ICmpInst::ICMP_UGT:
7119 NewPred = ICmpInst::ICMP_EQ;
7120 break;
7121 default:
7122 return nullptr;
7123 }
7124 } else if (match(&Cmp, m_c_ICmp(Pred,
7127 m_Add(m_Shl(m_One(), m_Value(Y)),
7128 m_AllOnes()))),
7129 m_Value(X)))) {
7130 // The variant with 'add' is not canonical, (the variant with 'not' is)
7131 // we only get it because it has extra uses, and can't be canonicalized,
7132
7133 switch (Pred) {
7134 case ICmpInst::ICMP_ULT:
7135 NewPred = ICmpInst::ICMP_NE;
7136 break;
7137 case ICmpInst::ICMP_UGE:
7138 NewPred = ICmpInst::ICMP_EQ;
7139 break;
7140 default:
7141 return nullptr;
7142 }
7143 } else
7144 return nullptr;
7145
7146 Value *NewX = Builder.CreateLShr(X, Y, X->getName() + ".highbits");
7147 Constant *Zero = Constant::getNullValue(NewX->getType());
7148 return CmpInst::Create(Instruction::ICmp, NewPred, NewX, Zero);
7149}
7150
7152 InstCombiner::BuilderTy &Builder) {
7153 const CmpInst::Predicate Pred = Cmp.getPredicate();
7154 Value *LHS = Cmp.getOperand(0), *RHS = Cmp.getOperand(1);
7155 Value *V1, *V2;
7156
7157 auto createCmpReverse = [&](CmpInst::Predicate Pred, Value *X, Value *Y) {
7158 Value *V = Builder.CreateCmp(Pred, X, Y, Cmp.getName());
7159 if (auto *I = dyn_cast<Instruction>(V))
7160 I->copyIRFlags(&Cmp);
7161 Module *M = Cmp.getModule();
7163 M, Intrinsic::vector_reverse, V->getType());
7164 return CallInst::Create(F, V);
7165 };
7166
7167 if (match(LHS, m_VecReverse(m_Value(V1)))) {
7168 // cmp Pred, rev(V1), rev(V2) --> rev(cmp Pred, V1, V2)
7169 if (match(RHS, m_VecReverse(m_Value(V2))) &&
7170 (LHS->hasOneUse() || RHS->hasOneUse()))
7171 return createCmpReverse(Pred, V1, V2);
7172
7173 // cmp Pred, rev(V1), RHSSplat --> rev(cmp Pred, V1, RHSSplat)
7174 if (LHS->hasOneUse() && isSplatValue(RHS))
7175 return createCmpReverse(Pred, V1, RHS);
7176 }
7177 // cmp Pred, LHSSplat, rev(V2) --> rev(cmp Pred, LHSSplat, V2)
7178 else if (isSplatValue(LHS) && match(RHS, m_OneUse(m_VecReverse(m_Value(V2)))))
7179 return createCmpReverse(Pred, LHS, V2);
7180
7181 ArrayRef<int> M;
7182 if (!match(LHS, m_Shuffle(m_Value(V1), m_Undef(), m_Mask(M))))
7183 return nullptr;
7184
7185 // If both arguments of the cmp are shuffles that use the same mask and
7186 // shuffle within a single vector, move the shuffle after the cmp:
7187 // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M
7188 Type *V1Ty = V1->getType();
7189 if (match(RHS, m_Shuffle(m_Value(V2), m_Undef(), m_SpecificMask(M))) &&
7190 V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) {
7191 Value *NewCmp = Builder.CreateCmp(Pred, V1, V2);
7192 return new ShuffleVectorInst(NewCmp, M);
7193 }
7194
7195 // Try to canonicalize compare with splatted operand and splat constant.
7196 // TODO: We could generalize this for more than splats. See/use the code in
7197 // InstCombiner::foldVectorBinop().
7198 Constant *C;
7199 if (!LHS->hasOneUse() || !match(RHS, m_Constant(C)))
7200 return nullptr;
7201
7202 // Length-changing splats are ok, so adjust the constants as needed:
7203 // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M
7204 Constant *ScalarC = C->getSplatValue(/* AllowPoison */ true);
7205 int MaskSplatIndex;
7206 if (ScalarC && match(M, m_SplatOrPoisonMask(MaskSplatIndex))) {
7207 // We allow poison in matching, but this transform removes it for safety.
7208 // Demanded elements analysis should be able to recover some/all of that.
7209 C = ConstantVector::getSplat(cast<VectorType>(V1Ty)->getElementCount(),
7210 ScalarC);
7211 SmallVector<int, 8> NewM(M.size(), MaskSplatIndex);
7212 Value *NewCmp = Builder.CreateCmp(Pred, V1, C);
7213 return new ShuffleVectorInst(NewCmp, NewM);
7214 }
7215
7216 return nullptr;
7217}
7218
7219// extract(uadd.with.overflow(A, B), 0) ult A
7220// -> extract(uadd.with.overflow(A, B), 1)
7222 CmpInst::Predicate Pred = I.getPredicate();
7223 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
7224
7225 Value *UAddOv;
7226 Value *A, *B;
7227 auto UAddOvResultPat = m_ExtractValue<0>(
7228 m_Intrinsic<Intrinsic::uadd_with_overflow>(m_Value(A), m_Value(B)));
7229 if (match(Op0, UAddOvResultPat) &&
7230 ((Pred == ICmpInst::ICMP_ULT && (Op1 == A || Op1 == B)) ||
7231 (Pred == ICmpInst::ICMP_EQ && match(Op1, m_ZeroInt()) &&
7232 (match(A, m_One()) || match(B, m_One()))) ||
7233 (Pred == ICmpInst::ICMP_NE && match(Op1, m_AllOnes()) &&
7234 (match(A, m_AllOnes()) || match(B, m_AllOnes())))))
7235 // extract(uadd.with.overflow(A, B), 0) < A
7236 // extract(uadd.with.overflow(A, 1), 0) == 0
7237 // extract(uadd.with.overflow(A, -1), 0) != -1
7238 UAddOv = cast<ExtractValueInst>(Op0)->getAggregateOperand();
7239 else if (match(Op1, UAddOvResultPat) &&
7240 Pred == ICmpInst::ICMP_UGT && (Op0 == A || Op0 == B))
7241 // A > extract(uadd.with.overflow(A, B), 0)
7242 UAddOv = cast<ExtractValueInst>(Op1)->getAggregateOperand();
7243 else
7244 return nullptr;
7245
7246 return ExtractValueInst::Create(UAddOv, 1);
7247}
7248
7250 if (!I.getOperand(0)->getType()->isPointerTy() ||
7252 I.getParent()->getParent(),
7253 I.getOperand(0)->getType()->getPointerAddressSpace())) {
7254 return nullptr;
7255 }
7256 Instruction *Op;
7257 if (match(I.getOperand(0), m_Instruction(Op)) &&
7258 match(I.getOperand(1), m_Zero()) &&
7259 Op->isLaunderOrStripInvariantGroup()) {
7260 return ICmpInst::Create(Instruction::ICmp, I.getPredicate(),
7261 Op->getOperand(0), I.getOperand(1));
7262 }
7263 return nullptr;
7264}
7265
7266/// This function folds patterns produced by lowering of reduce idioms, such as
7267/// llvm.vector.reduce.and which are lowered into instruction chains. This code
7268/// attempts to generate fewer number of scalar comparisons instead of vector
7269/// comparisons when possible.
7271 InstCombiner::BuilderTy &Builder,
7272 const DataLayout &DL) {
7273 if (I.getType()->isVectorTy())
7274 return nullptr;
7275 CmpPredicate OuterPred, InnerPred;
7276 Value *LHS, *RHS;
7277
7278 // Match lowering of @llvm.vector.reduce.and. Turn
7279 /// %vec_ne = icmp ne <8 x i8> %lhs, %rhs
7280 /// %scalar_ne = bitcast <8 x i1> %vec_ne to i8
7281 /// %res = icmp <pred> i8 %scalar_ne, 0
7282 ///
7283 /// into
7284 ///
7285 /// %lhs.scalar = bitcast <8 x i8> %lhs to i64
7286 /// %rhs.scalar = bitcast <8 x i8> %rhs to i64
7287 /// %res = icmp <pred> i64 %lhs.scalar, %rhs.scalar
7288 ///
7289 /// for <pred> in {ne, eq}.
7290 if (!match(&I, m_ICmp(OuterPred,
7292 m_ICmp(InnerPred, m_Value(LHS), m_Value(RHS))))),
7293 m_Zero())))
7294 return nullptr;
7295 auto *LHSTy = dyn_cast<FixedVectorType>(LHS->getType());
7296 if (!LHSTy || !LHSTy->getElementType()->isIntegerTy())
7297 return nullptr;
7298 unsigned NumBits =
7299 LHSTy->getNumElements() * LHSTy->getElementType()->getIntegerBitWidth();
7300 // TODO: Relax this to "not wider than max legal integer type"?
7301 if (!DL.isLegalInteger(NumBits))
7302 return nullptr;
7303
7304 if (ICmpInst::isEquality(OuterPred) && InnerPred == ICmpInst::ICMP_NE) {
7305 auto *ScalarTy = Builder.getIntNTy(NumBits);
7306 LHS = Builder.CreateBitCast(LHS, ScalarTy, LHS->getName() + ".scalar");
7307 RHS = Builder.CreateBitCast(RHS, ScalarTy, RHS->getName() + ".scalar");
7308 return ICmpInst::Create(Instruction::ICmp, OuterPred, LHS, RHS,
7309 I.getName());
7310 }
7311
7312 return nullptr;
7313}
7314
7315// This helper will be called with icmp operands in both orders.
7317 Value *Op0, Value *Op1,
7318 ICmpInst &CxtI) {
7319 // Try to optimize 'icmp GEP, P' or 'icmp P, GEP'.
7320 if (auto *GEP = dyn_cast<GEPOperator>(Op0))
7321 if (Instruction *NI = foldGEPICmp(GEP, Op1, Pred, CxtI))
7322 return NI;
7323
7324 if (auto *SI = dyn_cast<SelectInst>(Op0))
7325 if (Instruction *NI = foldSelectICmp(Pred, SI, Op1, CxtI))
7326 return NI;
7327
7328 if (auto *MinMax = dyn_cast<MinMaxIntrinsic>(Op0))
7329 if (Instruction *Res = foldICmpWithMinMax(CxtI, MinMax, Op1, Pred))
7330 return Res;
7331
7332 {
7333 Value *X;
7334 const APInt *C;
7335 // icmp X+Cst, X
7336 if (match(Op0, m_Add(m_Value(X), m_APInt(C))) && Op1 == X)
7337 return foldICmpAddOpConst(X, *C, Pred);
7338 }
7339
7340 // abs(X) >= X --> true
7341 // abs(X) u<= X --> true
7342 // abs(X) < X --> false
7343 // abs(X) u> X --> false
7344 // abs(X) u>= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
7345 // abs(X) <= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
7346 // abs(X) == X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
7347 // abs(X) u< X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
7348 // abs(X) > X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
7349 // abs(X) != X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
7350 {
7351 Value *X;
7352 Constant *C;
7353 if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(X), m_Constant(C))) &&
7354 match(Op1, m_Specific(X))) {
7355 Value *NullValue = Constant::getNullValue(X->getType());
7356 Value *AllOnesValue = Constant::getAllOnesValue(X->getType());
7357 const APInt SMin =
7358 APInt::getSignedMinValue(X->getType()->getScalarSizeInBits());
7359 bool IsIntMinPosion = C->isAllOnesValue();
7360 switch (Pred) {
7361 case CmpInst::ICMP_ULE:
7362 case CmpInst::ICMP_SGE:
7363 return replaceInstUsesWith(CxtI, ConstantInt::getTrue(CxtI.getType()));
7364 case CmpInst::ICMP_UGT:
7365 case CmpInst::ICMP_SLT:
7367 case CmpInst::ICMP_UGE:
7368 case CmpInst::ICMP_SLE:
7369 case CmpInst::ICMP_EQ: {
7370 return replaceInstUsesWith(
7371 CxtI, IsIntMinPosion
7372 ? Builder.CreateICmpSGT(X, AllOnesValue)
7374 X, ConstantInt::get(X->getType(), SMin + 1)));
7375 }
7376 case CmpInst::ICMP_ULT:
7377 case CmpInst::ICMP_SGT:
7378 case CmpInst::ICMP_NE: {
7379 return replaceInstUsesWith(
7380 CxtI, IsIntMinPosion
7381 ? Builder.CreateICmpSLT(X, NullValue)
7383 X, ConstantInt::get(X->getType(), SMin)));
7384 }
7385 default:
7386 llvm_unreachable("Invalid predicate!");
7387 }
7388 }
7389 }
7390
7391 const SimplifyQuery Q = SQ.getWithInstruction(&CxtI);
7392 if (Value *V = foldICmpWithLowBitMaskedVal(Pred, Op0, Op1, Q, *this))
7393 return replaceInstUsesWith(CxtI, V);
7394
7395 // Folding (X / Y) pred X => X swap(pred) 0 for constant Y other than 0 or 1
7396 auto CheckUGT1 = [](const APInt &Divisor) { return Divisor.ugt(1); };
7397 {
7398 if (match(Op0, m_UDiv(m_Specific(Op1), m_CheckedInt(CheckUGT1)))) {
7399 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
7401 }
7402
7403 if (!ICmpInst::isUnsigned(Pred) &&
7404 match(Op0, m_SDiv(m_Specific(Op1), m_CheckedInt(CheckUGT1)))) {
7405 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
7407 }
7408 }
7409
7410 // Another case of this fold is (X >> Y) pred X => X swap(pred) 0 if Y != 0
7411 auto CheckNE0 = [](const APInt &Shift) { return !Shift.isZero(); };
7412 {
7413 if (match(Op0, m_LShr(m_Specific(Op1), m_CheckedInt(CheckNE0)))) {
7414 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
7416 }
7417
7418 if ((Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SGE) &&
7419 match(Op0, m_AShr(m_Specific(Op1), m_CheckedInt(CheckNE0)))) {
7420 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
7422 }
7423 }
7424
7425 return nullptr;
7426}
7427
7429 bool Changed = false;
7431 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
7432 unsigned Op0Cplxity = getComplexity(Op0);
7433 unsigned Op1Cplxity = getComplexity(Op1);
7434
7435 /// Orders the operands of the compare so that they are listed from most
7436 /// complex to least complex. This puts constants before unary operators,
7437 /// before binary operators.
7438 if (Op0Cplxity < Op1Cplxity) {
7439 I.swapOperands();
7440 std::swap(Op0, Op1);
7441 Changed = true;
7442 }
7443
7444 if (Value *V = simplifyICmpInst(I.getCmpPredicate(), Op0, Op1, Q))
7445 return replaceInstUsesWith(I, V);
7446
7447 // Comparing -val or val with non-zero is the same as just comparing val
7448 // ie, abs(val) != 0 -> val != 0
7449 if (I.getPredicate() == ICmpInst::ICMP_NE && match(Op1, m_Zero())) {
7450 Value *Cond, *SelectTrue, *SelectFalse;
7451 if (match(Op0, m_Select(m_Value(Cond), m_Value(SelectTrue),
7452 m_Value(SelectFalse)))) {
7453 if (Value *V = dyn_castNegVal(SelectTrue)) {
7454 if (V == SelectFalse)
7455 return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1);
7456 }
7457 else if (Value *V = dyn_castNegVal(SelectFalse)) {
7458 if (V == SelectTrue)
7459 return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1);
7460 }
7461 }
7462 }
7463
7464 if (Op0->getType()->isIntOrIntVectorTy(1))
7466 return Res;
7467
7469 return Res;
7470
7472 return Res;
7473
7475 return Res;
7476
7478 return Res;
7479
7481 return Res;
7482
7484 return Res;
7485
7487 return Res;
7488
7489 // Test if the ICmpInst instruction is used exclusively by a select as
7490 // part of a minimum or maximum operation. If so, refrain from doing
7491 // any other folding. This helps out other analyses which understand
7492 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
7493 // and CodeGen. And in this case, at least one of the comparison
7494 // operands has at least one user besides the compare (the select),
7495 // which would often largely negate the benefit of folding anyway.
7496 //
7497 // Do the same for the other patterns recognized by matchSelectPattern.
7498 if (I.hasOneUse())
7499 if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) {
7500 Value *A, *B;
7502 if (SPR.Flavor != SPF_UNKNOWN)
7503 return nullptr;
7504 }
7505
7506 // Do this after checking for min/max to prevent infinite looping.
7507 if (Instruction *Res = foldICmpWithZero(I))
7508 return Res;
7509
7510 // FIXME: We only do this after checking for min/max to prevent infinite
7511 // looping caused by a reverse canonicalization of these patterns for min/max.
7512 // FIXME: The organization of folds is a mess. These would naturally go into
7513 // canonicalizeCmpWithConstant(), but we can't move all of the above folds
7514 // down here after the min/max restriction.
7515 ICmpInst::Predicate Pred = I.getPredicate();
7516 const APInt *C;
7517 if (match(Op1, m_APInt(C))) {
7518 // For i32: x >u 2147483647 -> x <s 0 -> true if sign bit set
7519 if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) {
7520 Constant *Zero = Constant::getNullValue(Op0->getType());
7521 return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero);
7522 }
7523
7524 // For i32: x <u 2147483648 -> x >s -1 -> true if sign bit clear
7525 if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) {
7527 return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes);
7528 }
7529 }
7530
7531 // The folds in here may rely on wrapping flags and special constants, so
7532 // they can break up min/max idioms in some cases but not seemingly similar
7533 // patterns.
7534 // FIXME: It may be possible to enhance select folding to make this
7535 // unnecessary. It may also be moot if we canonicalize to min/max
7536 // intrinsics.
7537 if (Instruction *Res = foldICmpBinOp(I, Q))
7538 return Res;
7539
7541 return Res;
7542
7543 // Try to match comparison as a sign bit test. Intentionally do this after
7544 // foldICmpInstWithConstant() to potentially let other folds to happen first.
7545 if (Instruction *New = foldSignBitTest(I))
7546 return New;
7547
7549 return Res;
7550
7551 if (Instruction *Res = foldICmpCommutative(I.getCmpPredicate(), Op0, Op1, I))
7552 return Res;
7553 if (Instruction *Res =
7554 foldICmpCommutative(I.getSwappedCmpPredicate(), Op1, Op0, I))
7555 return Res;
7556
7557 if (I.isCommutative()) {
7558 if (auto Pair = matchSymmetricPair(I.getOperand(0), I.getOperand(1))) {
7559 replaceOperand(I, 0, Pair->first);
7560 replaceOperand(I, 1, Pair->second);
7561 return &I;
7562 }
7563 }
7564
7565 // In case of a comparison with two select instructions having the same
7566 // condition, check whether one of the resulting branches can be simplified.
7567 // If so, just compare the other branch and select the appropriate result.
7568 // For example:
7569 // %tmp1 = select i1 %cmp, i32 %y, i32 %x
7570 // %tmp2 = select i1 %cmp, i32 %z, i32 %x
7571 // %cmp2 = icmp slt i32 %tmp2, %tmp1
7572 // The icmp will result false for the false value of selects and the result
7573 // will depend upon the comparison of true values of selects if %cmp is
7574 // true. Thus, transform this into:
7575 // %cmp = icmp slt i32 %y, %z
7576 // %sel = select i1 %cond, i1 %cmp, i1 false
7577 // This handles similar cases to transform.
7578 {
7579 Value *Cond, *A, *B, *C, *D;
7580 if (match(Op0, m_Select(m_Value(Cond), m_Value(A), m_Value(B))) &&
7582 (Op0->hasOneUse() || Op1->hasOneUse())) {
7583 // Check whether comparison of TrueValues can be simplified
7584 if (Value *Res = simplifyICmpInst(Pred, A, C, SQ)) {
7585 Value *NewICMP = Builder.CreateICmp(Pred, B, D);
7586 return SelectInst::Create(Cond, Res, NewICMP);
7587 }
7588 // Check whether comparison of FalseValues can be simplified
7589 if (Value *Res = simplifyICmpInst(Pred, B, D, SQ)) {
7590 Value *NewICMP = Builder.CreateICmp(Pred, A, C);
7591 return SelectInst::Create(Cond, NewICMP, Res);
7592 }
7593 }
7594 }
7595
7596 // Try to optimize equality comparisons against alloca-based pointers.
7597 if (Op0->getType()->isPointerTy() && I.isEquality()) {
7598 assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?");
7599 if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0)))
7600 if (foldAllocaCmp(Alloca))
7601 return nullptr;
7602 if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1)))
7603 if (foldAllocaCmp(Alloca))
7604 return nullptr;
7605 }
7606
7607 if (Instruction *Res = foldICmpBitCast(I))
7608 return Res;
7609
7610 // TODO: Hoist this above the min/max bailout.
7612 return R;
7613
7614 {
7615 Value *X, *Y;
7616 // Transform (X & ~Y) == 0 --> (X & Y) != 0
7617 // and (X & ~Y) != 0 --> (X & Y) == 0
7618 // if A is a power of 2.
7619 if (match(Op0, m_And(m_Value(X), m_Not(m_Value(Y)))) &&
7620 match(Op1, m_Zero()) && isKnownToBeAPowerOfTwo(X, false, 0, &I) &&
7621 I.isEquality())
7622 return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(X, Y),
7623 Op1);
7624
7625 // Op0 pred Op1 -> ~Op1 pred ~Op0, if this allows us to drop an instruction.
7626 if (Op0->getType()->isIntOrIntVectorTy()) {
7627 bool ConsumesOp0, ConsumesOp1;
7628 if (isFreeToInvert(Op0, Op0->hasOneUse(), ConsumesOp0) &&
7629 isFreeToInvert(Op1, Op1->hasOneUse(), ConsumesOp1) &&
7630 (ConsumesOp0 || ConsumesOp1)) {
7631 Value *InvOp0 = getFreelyInverted(Op0, Op0->hasOneUse(), &Builder);
7632 Value *InvOp1 = getFreelyInverted(Op1, Op1->hasOneUse(), &Builder);
7633 assert(InvOp0 && InvOp1 &&
7634 "Mismatch between isFreeToInvert and getFreelyInverted");
7635 return new ICmpInst(I.getSwappedPredicate(), InvOp0, InvOp1);
7636 }
7637 }
7638
7639 Instruction *AddI = nullptr;
7641 m_Instruction(AddI))) &&
7642 isa<IntegerType>(X->getType())) {
7643 Value *Result;
7644 Constant *Overflow;
7645 // m_UAddWithOverflow can match patterns that do not include an explicit
7646 // "add" instruction, so check the opcode of the matched op.
7647 if (AddI->getOpcode() == Instruction::Add &&
7648 OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, X, Y, *AddI,
7649 Result, Overflow)) {
7650 replaceInstUsesWith(*AddI, Result);
7651 eraseInstFromFunction(*AddI);
7652 return replaceInstUsesWith(I, Overflow);
7653 }
7654 }
7655
7656 // (zext X) * (zext Y) --> llvm.umul.with.overflow.
7657 if (match(Op0, m_NUWMul(m_ZExt(m_Value(X)), m_ZExt(m_Value(Y)))) &&
7658 match(Op1, m_APInt(C))) {
7659 if (Instruction *R = processUMulZExtIdiom(I, Op0, C, *this))
7660 return R;
7661 }
7662
7663 // Signbit test folds
7664 // Fold (X u>> BitWidth - 1 Pred ZExt(i1)) --> X s< 0 Pred i1
7665 // Fold (X s>> BitWidth - 1 Pred SExt(i1)) --> X s< 0 Pred i1
7666 Instruction *ExtI;
7667 if ((I.isUnsigned() || I.isEquality()) &&
7668 match(Op1,
7670 Y->getType()->getScalarSizeInBits() == 1 &&
7671 (Op0->hasOneUse() || Op1->hasOneUse())) {
7672 unsigned OpWidth = Op0->getType()->getScalarSizeInBits();
7673 Instruction *ShiftI;
7674 if (match(Op0, m_CombineAnd(m_Instruction(ShiftI),
7676 OpWidth - 1))))) {
7677 unsigned ExtOpc = ExtI->getOpcode();
7678 unsigned ShiftOpc = ShiftI->getOpcode();
7679 if ((ExtOpc == Instruction::ZExt && ShiftOpc == Instruction::LShr) ||
7680 (ExtOpc == Instruction::SExt && ShiftOpc == Instruction::AShr)) {
7681 Value *SLTZero =
7683 Value *Cmp = Builder.CreateICmp(Pred, SLTZero, Y, I.getName());
7684 return replaceInstUsesWith(I, Cmp);
7685 }
7686 }
7687 }
7688 }
7689
7690 if (Instruction *Res = foldICmpEquality(I))
7691 return Res;
7692
7694 return Res;
7695
7696 if (Instruction *Res = foldICmpOfUAddOv(I))
7697 return Res;
7698
7699 // The 'cmpxchg' instruction returns an aggregate containing the old value and
7700 // an i1 which indicates whether or not we successfully did the swap.
7701 //
7702 // Replace comparisons between the old value and the expected value with the
7703 // indicator that 'cmpxchg' returns.
7704 //
7705 // N.B. This transform is only valid when the 'cmpxchg' is not permitted to
7706 // spuriously fail. In those cases, the old value may equal the expected
7707 // value but it is possible for the swap to not occur.
7708 if (I.getPredicate() == ICmpInst::ICMP_EQ)
7709 if (auto *EVI = dyn_cast<ExtractValueInst>(Op0))
7710 if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(EVI->getAggregateOperand()))
7711 if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 &&
7712 !ACXI->isWeak())
7713 return ExtractValueInst::Create(ACXI, 1);
7714
7716 return Res;
7717
7718 if (I.getType()->isVectorTy())
7719 if (Instruction *Res = foldVectorCmp(I, Builder))
7720 return Res;
7721
7723 return Res;
7724
7726 return Res;
7727
7728 {
7729 Value *A;
7730 const APInt *C1, *C2;
7731 ICmpInst::Predicate Pred = I.getPredicate();
7732 if (ICmpInst::isEquality(Pred)) {
7733 // sext(a) & c1 == c2 --> a & c3 == trunc(c2)
7734 // sext(a) & c1 != c2 --> a & c3 != trunc(c2)
7735 if (match(Op0, m_And(m_SExt(m_Value(A)), m_APInt(C1))) &&
7736 match(Op1, m_APInt(C2))) {
7737 Type *InputTy = A->getType();
7738 unsigned InputBitWidth = InputTy->getScalarSizeInBits();
7739 // c2 must be non-negative at the bitwidth of a.
7740 if (C2->getActiveBits() < InputBitWidth) {
7741 APInt TruncC1 = C1->trunc(InputBitWidth);
7742 // Check if there are 1s in C1 high bits of size InputBitWidth.
7743 if (C1->uge(APInt::getOneBitSet(C1->getBitWidth(), InputBitWidth)))
7744 TruncC1.setBit(InputBitWidth - 1);
7745 Value *AndInst = Builder.CreateAnd(A, TruncC1);
7746 return new ICmpInst(
7747 Pred, AndInst,
7748 ConstantInt::get(InputTy, C2->trunc(InputBitWidth)));
7749 }
7750 }
7751 }
7752 }
7753
7754 return Changed ? &I : nullptr;
7755}
7756
7757/// Fold fcmp ([us]itofp x, cst) if possible.
7759 Instruction *LHSI,
7760 Constant *RHSC) {
7761 const APFloat *RHS;
7762 if (!match(RHSC, m_APFloat(RHS)))
7763 return nullptr;
7764
7765 // Get the width of the mantissa. We don't want to hack on conversions that
7766 // might lose information from the integer, e.g. "i64 -> float"
7767 int MantissaWidth = LHSI->getType()->getFPMantissaWidth();
7768 if (MantissaWidth == -1) return nullptr; // Unknown.
7769
7770 Type *IntTy = LHSI->getOperand(0)->getType();
7771 unsigned IntWidth = IntTy->getScalarSizeInBits();
7772 bool LHSUnsigned = isa<UIToFPInst>(LHSI);
7773
7774 if (I.isEquality()) {
7775 FCmpInst::Predicate P = I.getPredicate();
7776 bool IsExact = false;
7777 APSInt RHSCvt(IntWidth, LHSUnsigned);
7778 RHS->convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact);
7779
7780 // If the floating point constant isn't an integer value, we know if we will
7781 // ever compare equal / not equal to it.
7782 if (!IsExact) {
7783 // TODO: Can never be -0.0 and other non-representable values
7784 APFloat RHSRoundInt(*RHS);
7786 if (*RHS != RHSRoundInt) {
7788 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7789
7791 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7792 }
7793 }
7794
7795 // TODO: If the constant is exactly representable, is it always OK to do
7796 // equality compares as integer?
7797 }
7798
7799 // Check to see that the input is converted from an integer type that is small
7800 // enough that preserves all bits. TODO: check here for "known" sign bits.
7801 // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e.
7802
7803 // Following test does NOT adjust IntWidth downwards for signed inputs,
7804 // because the most negative value still requires all the mantissa bits
7805 // to distinguish it from one less than that value.
7806 if ((int)IntWidth > MantissaWidth) {
7807 // Conversion would lose accuracy. Check if loss can impact comparison.
7808 int Exp = ilogb(*RHS);
7809 if (Exp == APFloat::IEK_Inf) {
7810 int MaxExponent = ilogb(APFloat::getLargest(RHS->getSemantics()));
7811 if (MaxExponent < (int)IntWidth - !LHSUnsigned)
7812 // Conversion could create infinity.
7813 return nullptr;
7814 } else {
7815 // Note that if RHS is zero or NaN, then Exp is negative
7816 // and first condition is trivially false.
7817 if (MantissaWidth <= Exp && Exp <= (int)IntWidth - !LHSUnsigned)
7818 // Conversion could affect comparison.
7819 return nullptr;
7820 }
7821 }
7822
7823 // Otherwise, we can potentially simplify the comparison. We know that it
7824 // will always come through as an integer value and we know the constant is
7825 // not a NAN (it would have been previously simplified).
7826 assert(!RHS->isNaN() && "NaN comparison not already folded!");
7827
7829 switch (I.getPredicate()) {
7830 default: llvm_unreachable("Unexpected predicate!");
7831 case FCmpInst::FCMP_UEQ:
7832 case FCmpInst::FCMP_OEQ:
7833 Pred = ICmpInst::ICMP_EQ;
7834 break;
7835 case FCmpInst::FCMP_UGT:
7836 case FCmpInst::FCMP_OGT:
7837 Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT;
7838 break;
7839 case FCmpInst::FCMP_UGE:
7840 case FCmpInst::FCMP_OGE:
7841 Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE;
7842 break;
7843 case FCmpInst::FCMP_ULT:
7844 case FCmpInst::FCMP_OLT:
7845 Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT;
7846 break;
7847 case FCmpInst::FCMP_ULE:
7848 case FCmpInst::FCMP_OLE:
7849 Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE;
7850 break;
7851 case FCmpInst::FCMP_UNE:
7852 case FCmpInst::FCMP_ONE:
7853 Pred = ICmpInst::ICMP_NE;
7854 break;
7855 case FCmpInst::FCMP_ORD:
7856 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7857 case FCmpInst::FCMP_UNO:
7858 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7859 }
7860
7861 // Now we know that the APFloat is a normal number, zero or inf.
7862
7863 // See if the FP constant is too large for the integer. For example,
7864 // comparing an i8 to 300.0.
7865 if (!LHSUnsigned) {
7866 // If the RHS value is > SignedMax, fold the comparison. This handles +INF
7867 // and large values.
7868 APFloat SMax(RHS->getSemantics());
7869 SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true,
7871 if (SMax < *RHS) { // smax < 13123.0
7872 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT ||
7873 Pred == ICmpInst::ICMP_SLE)
7874 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7875 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7876 }
7877 } else {
7878 // If the RHS value is > UnsignedMax, fold the comparison. This handles
7879 // +INF and large values.
7880 APFloat UMax(RHS->getSemantics());
7881 UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false,
7883 if (UMax < *RHS) { // umax < 13123.0
7884 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT ||
7885 Pred == ICmpInst::ICMP_ULE)
7886 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7887 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7888 }
7889 }
7890
7891 if (!LHSUnsigned) {
7892 // See if the RHS value is < SignedMin.
7893 APFloat SMin(RHS->getSemantics());
7894 SMin.convertFromAPInt(APInt::getSignedMinValue(IntWidth), true,
7896 if (SMin > *RHS) { // smin > 12312.0
7897 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT ||
7898 Pred == ICmpInst::ICMP_SGE)
7899 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7900 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7901 }
7902 } else {
7903 // See if the RHS value is < UnsignedMin.
7904 APFloat UMin(RHS->getSemantics());
7905 UMin.convertFromAPInt(APInt::getMinValue(IntWidth), false,
7907 if (UMin > *RHS) { // umin > 12312.0
7908 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT ||
7909 Pred == ICmpInst::ICMP_UGE)
7910 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7911 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7912 }
7913 }
7914
7915 // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or
7916 // [0, UMAX], but it may still be fractional. Check whether this is the case
7917 // using the IsExact flag.
7918 // Don't do this for zero, because -0.0 is not fractional.
7919 APSInt RHSInt(IntWidth, LHSUnsigned);
7920 bool IsExact;
7921 RHS->convertToInteger(RHSInt, APFloat::rmTowardZero, &IsExact);
7922 if (!RHS->isZero()) {
7923 if (!IsExact) {
7924 // If we had a comparison against a fractional value, we have to adjust
7925 // the compare predicate and sometimes the value. RHSC is rounded towards
7926 // zero at this point.
7927 switch (Pred) {
7928 default: llvm_unreachable("Unexpected integer comparison!");
7929 case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true
7930 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7931 case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false
7932 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7933 case ICmpInst::ICMP_ULE:
7934 // (float)int <= 4.4 --> int <= 4
7935 // (float)int <= -4.4 --> false
7936 if (RHS->isNegative())
7937 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7938 break;
7939 case ICmpInst::ICMP_SLE:
7940 // (float)int <= 4.4 --> int <= 4
7941 // (float)int <= -4.4 --> int < -4
7942 if (RHS->isNegative())
7943 Pred = ICmpInst::ICMP_SLT;
7944 break;
7945 case ICmpInst::ICMP_ULT:
7946 // (float)int < -4.4 --> false
7947 // (float)int < 4.4 --> int <= 4
7948 if (RHS->isNegative())
7949 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7950 Pred = ICmpInst::ICMP_ULE;
7951 break;
7952 case ICmpInst::ICMP_SLT:
7953 // (float)int < -4.4 --> int < -4
7954 // (float)int < 4.4 --> int <= 4
7955 if (!RHS->isNegative())
7956 Pred = ICmpInst::ICMP_SLE;
7957 break;
7958 case ICmpInst::ICMP_UGT:
7959 // (float)int > 4.4 --> int > 4
7960 // (float)int > -4.4 --> true
7961 if (RHS->isNegative())
7962 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7963 break;
7964 case ICmpInst::ICMP_SGT:
7965 // (float)int > 4.4 --> int > 4
7966 // (float)int > -4.4 --> int >= -4
7967 if (RHS->isNegative())
7968 Pred = ICmpInst::ICMP_SGE;
7969 break;
7970 case ICmpInst::ICMP_UGE:
7971 // (float)int >= -4.4 --> true
7972 // (float)int >= 4.4 --> int > 4
7973 if (RHS->isNegative())
7974 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7975 Pred = ICmpInst::ICMP_UGT;
7976 break;
7977 case ICmpInst::ICMP_SGE:
7978 // (float)int >= -4.4 --> int >= -4
7979 // (float)int >= 4.4 --> int > 4
7980 if (!RHS->isNegative())
7981 Pred = ICmpInst::ICMP_SGT;
7982 break;
7983 }
7984 }
7985 }
7986
7987 // Lower this FP comparison into an appropriate integer version of the
7988 // comparison.
7989 return new ICmpInst(Pred, LHSI->getOperand(0),
7990 ConstantInt::get(LHSI->getOperand(0)->getType(), RHSInt));
7991}
7992
7993/// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary.
7995 Constant *RHSC) {
7996 // When C is not 0.0 and infinities are not allowed:
7997 // (C / X) < 0.0 is a sign-bit test of X
7998 // (C / X) < 0.0 --> X < 0.0 (if C is positive)
7999 // (C / X) < 0.0 --> X > 0.0 (if C is negative, swap the predicate)
8000 //
8001 // Proof:
8002 // Multiply (C / X) < 0.0 by X * X / C.
8003 // - X is non zero, if it is the flag 'ninf' is violated.
8004 // - C defines the sign of X * X * C. Thus it also defines whether to swap
8005 // the predicate. C is also non zero by definition.
8006 //
8007 // Thus X * X / C is non zero and the transformation is valid. [qed]
8008
8009 FCmpInst::Predicate Pred = I.getPredicate();
8010
8011 // Check that predicates are valid.
8012 if ((Pred != FCmpInst::FCMP_OGT) && (Pred != FCmpInst::FCMP_OLT) &&
8013 (Pred != FCmpInst::FCMP_OGE) && (Pred != FCmpInst::FCMP_OLE))
8014 return nullptr;
8015
8016 // Check that RHS operand is zero.
8017 if (!match(RHSC, m_AnyZeroFP()))
8018 return nullptr;
8019
8020 // Check fastmath flags ('ninf').
8021 if (!LHSI->hasNoInfs() || !I.hasNoInfs())
8022 return nullptr;
8023
8024 // Check the properties of the dividend. It must not be zero to avoid a
8025 // division by zero (see Proof).
8026 const APFloat *C;
8027 if (!match(LHSI->getOperand(0), m_APFloat(C)))
8028 return nullptr;
8029
8030 if (C->isZero())
8031 return nullptr;
8032
8033 // Get swapped predicate if necessary.
8034 if (C->isNegative())
8035 Pred = I.getSwappedPredicate();
8036
8037 return new FCmpInst(Pred, LHSI->getOperand(1), RHSC, "", &I);
8038}
8039
8040/// Optimize fabs(X) compared with zero.
8042 Value *X;
8043 if (!match(I.getOperand(0), m_FAbs(m_Value(X))))
8044 return nullptr;
8045
8046 const APFloat *C;
8047 if (!match(I.getOperand(1), m_APFloat(C)))
8048 return nullptr;
8049
8050 if (!C->isPosZero()) {
8051 if (!C->isSmallestNormalized())
8052 return nullptr;
8053
8054 const Function *F = I.getFunction();
8055 DenormalMode Mode = F->getDenormalMode(C->getSemantics());
8056 if (Mode.Input == DenormalMode::PreserveSign ||
8057 Mode.Input == DenormalMode::PositiveZero) {
8058
8059 auto replaceFCmp = [](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
8060 Constant *Zero = ConstantFP::getZero(X->getType());
8061 return new FCmpInst(P, X, Zero, "", I);
8062 };
8063
8064 switch (I.getPredicate()) {
8065 case FCmpInst::FCMP_OLT:
8066 // fcmp olt fabs(x), smallest_normalized_number -> fcmp oeq x, 0.0
8067 return replaceFCmp(&I, FCmpInst::FCMP_OEQ, X);
8068 case FCmpInst::FCMP_UGE:
8069 // fcmp uge fabs(x), smallest_normalized_number -> fcmp une x, 0.0
8070 return replaceFCmp(&I, FCmpInst::FCMP_UNE, X);
8071 case FCmpInst::FCMP_OGE:
8072 // fcmp oge fabs(x), smallest_normalized_number -> fcmp one x, 0.0
8073 return replaceFCmp(&I, FCmpInst::FCMP_ONE, X);
8074 case FCmpInst::FCMP_ULT:
8075 // fcmp ult fabs(x), smallest_normalized_number -> fcmp ueq x, 0.0
8076 return replaceFCmp(&I, FCmpInst::FCMP_UEQ, X);
8077 default:
8078 break;
8079 }
8080 }
8081
8082 return nullptr;
8083 }
8084
8085 auto replacePredAndOp0 = [&IC](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
8086 I->setPredicate(P);
8087 return IC.replaceOperand(*I, 0, X);
8088 };
8089
8090 switch (I.getPredicate()) {
8091 case FCmpInst::FCMP_UGE:
8092 case FCmpInst::FCMP_OLT:
8093 // fabs(X) >= 0.0 --> true
8094 // fabs(X) < 0.0 --> false
8095 llvm_unreachable("fcmp should have simplified");
8096
8097 case FCmpInst::FCMP_OGT:
8098 // fabs(X) > 0.0 --> X != 0.0
8099 return replacePredAndOp0(&I, FCmpInst::FCMP_ONE, X);
8100
8101 case FCmpInst::FCMP_UGT:
8102 // fabs(X) u> 0.0 --> X u!= 0.0
8103 return replacePredAndOp0(&I, FCmpInst::FCMP_UNE, X);
8104
8105 case FCmpInst::FCMP_OLE:
8106 // fabs(X) <= 0.0 --> X == 0.0
8107 return replacePredAndOp0(&I, FCmpInst::FCMP_OEQ, X);
8108
8109 case FCmpInst::FCMP_ULE:
8110 // fabs(X) u<= 0.0 --> X u== 0.0
8111 return replacePredAndOp0(&I, FCmpInst::FCMP_UEQ, X);
8112
8113 case FCmpInst::FCMP_OGE:
8114 // fabs(X) >= 0.0 --> !isnan(X)
8115 assert(!I.hasNoNaNs() && "fcmp should have simplified");
8116 return replacePredAndOp0(&I, FCmpInst::FCMP_ORD, X);
8117
8118 case FCmpInst::FCMP_ULT:
8119 // fabs(X) u< 0.0 --> isnan(X)
8120 assert(!I.hasNoNaNs() && "fcmp should have simplified");
8121 return replacePredAndOp0(&I, FCmpInst::FCMP_UNO, X);
8122
8123 case FCmpInst::FCMP_OEQ:
8124 case FCmpInst::FCMP_UEQ:
8125 case FCmpInst::FCMP_ONE:
8126 case FCmpInst::FCMP_UNE:
8127 case FCmpInst::FCMP_ORD:
8128 case FCmpInst::FCMP_UNO:
8129 // Look through the fabs() because it doesn't change anything but the sign.
8130 // fabs(X) == 0.0 --> X == 0.0,
8131 // fabs(X) != 0.0 --> X != 0.0
8132 // isnan(fabs(X)) --> isnan(X)
8133 // !isnan(fabs(X) --> !isnan(X)
8134 return replacePredAndOp0(&I, I.getPredicate(), X);
8135
8136 default:
8137 return nullptr;
8138 }
8139}
8140
8141/// Optimize sqrt(X) compared with zero.
8143 Value *X;
8144 if (!match(I.getOperand(0), m_Sqrt(m_Value(X))))
8145 return nullptr;
8146
8147 if (!match(I.getOperand(1), m_PosZeroFP()))
8148 return nullptr;
8149
8150 auto ReplacePredAndOp0 = [&](FCmpInst::Predicate P) {
8151 I.setPredicate(P);
8152 return IC.replaceOperand(I, 0, X);
8153 };
8154
8155 // Clear ninf flag if sqrt doesn't have it.
8156 if (!cast<Instruction>(I.getOperand(0))->hasNoInfs())
8157 I.setHasNoInfs(false);
8158
8159 switch (I.getPredicate()) {
8160 case FCmpInst::FCMP_OLT:
8161 case FCmpInst::FCMP_UGE:
8162 // sqrt(X) < 0.0 --> false
8163 // sqrt(X) u>= 0.0 --> true
8164 llvm_unreachable("fcmp should have simplified");
8165 case FCmpInst::FCMP_ULT:
8166 case FCmpInst::FCMP_ULE:
8167 case FCmpInst::FCMP_OGT:
8168 case FCmpInst::FCMP_OGE:
8169 case FCmpInst::FCMP_OEQ:
8170 case FCmpInst::FCMP_UNE:
8171 // sqrt(X) u< 0.0 --> X u< 0.0
8172 // sqrt(X) u<= 0.0 --> X u<= 0.0
8173 // sqrt(X) > 0.0 --> X > 0.0
8174 // sqrt(X) >= 0.0 --> X >= 0.0
8175 // sqrt(X) == 0.0 --> X == 0.0
8176 // sqrt(X) u!= 0.0 --> X u!= 0.0
8177 return IC.replaceOperand(I, 0, X);
8178
8179 case FCmpInst::FCMP_OLE:
8180 // sqrt(X) <= 0.0 --> X == 0.0
8181 return ReplacePredAndOp0(FCmpInst::FCMP_OEQ);
8182 case FCmpInst::FCMP_UGT:
8183 // sqrt(X) u> 0.0 --> X u!= 0.0
8184 return ReplacePredAndOp0(FCmpInst::FCMP_UNE);
8185 case FCmpInst::FCMP_UEQ:
8186 // sqrt(X) u== 0.0 --> X u<= 0.0
8187 return ReplacePredAndOp0(FCmpInst::FCMP_ULE);
8188 case FCmpInst::FCMP_ONE:
8189 // sqrt(X) != 0.0 --> X > 0.0
8190 return ReplacePredAndOp0(FCmpInst::FCMP_OGT);
8191 case FCmpInst::FCMP_ORD:
8192 // !isnan(sqrt(X)) --> X >= 0.0
8193 return ReplacePredAndOp0(FCmpInst::FCMP_OGE);
8194 case FCmpInst::FCMP_UNO:
8195 // isnan(sqrt(X)) --> X u< 0.0
8196 return ReplacePredAndOp0(FCmpInst::FCMP_ULT);
8197 default:
8198 llvm_unreachable("Unexpected predicate!");
8199 }
8200}
8201
8203 CmpInst::Predicate Pred = I.getPredicate();
8204 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
8205
8206 // Canonicalize fneg as Op1.
8207 if (match(Op0, m_FNeg(m_Value())) && !match(Op1, m_FNeg(m_Value()))) {
8208 std::swap(Op0, Op1);
8209 Pred = I.getSwappedPredicate();
8210 }
8211
8212 if (!match(Op1, m_FNeg(m_Specific(Op0))))
8213 return nullptr;
8214
8215 // Replace the negated operand with 0.0:
8216 // fcmp Pred Op0, -Op0 --> fcmp Pred Op0, 0.0
8217 Constant *Zero = ConstantFP::getZero(Op0->getType());
8218 return new FCmpInst(Pred, Op0, Zero, "", &I);
8219}
8220
8222 Constant *RHSC, InstCombinerImpl &CI) {
8223 const CmpInst::Predicate Pred = I.getPredicate();
8224 Value *X = LHSI->getOperand(0);
8225 Value *Y = LHSI->getOperand(1);
8226 switch (Pred) {
8227 default:
8228 break;
8229 case FCmpInst::FCMP_UGT:
8230 case FCmpInst::FCMP_ULT:
8231 case FCmpInst::FCMP_UNE:
8232 case FCmpInst::FCMP_OEQ:
8233 case FCmpInst::FCMP_OGE:
8234 case FCmpInst::FCMP_OLE:
8235 // The optimization is not valid if X and Y are infinities of the same
8236 // sign, i.e. the inf - inf = nan case. If the fsub has the ninf or nnan
8237 // flag then we can assume we do not have that case. Otherwise we might be
8238 // able to prove that either X or Y is not infinity.
8239 if (!LHSI->hasNoNaNs() && !LHSI->hasNoInfs() &&
8240 !isKnownNeverInfinity(Y, /*Depth=*/0,
8242 !isKnownNeverInfinity(X, /*Depth=*/0,
8244 break;
8245
8246 [[fallthrough]];
8247 case FCmpInst::FCMP_OGT:
8248 case FCmpInst::FCMP_OLT:
8249 case FCmpInst::FCMP_ONE:
8250 case FCmpInst::FCMP_UEQ:
8251 case FCmpInst::FCMP_UGE:
8252 case FCmpInst::FCMP_ULE:
8253 // fcmp pred (x - y), 0 --> fcmp pred x, y
8254 if (match(RHSC, m_AnyZeroFP()) &&
8255 I.getFunction()->getDenormalMode(
8256 LHSI->getType()->getScalarType()->getFltSemantics()) ==
8258 CI.replaceOperand(I, 0, X);
8259 CI.replaceOperand(I, 1, Y);
8260 return &I;
8261 }
8262 break;
8263 }
8264
8265 return nullptr;
8266}
8267
8269 InstCombinerImpl &IC) {
8270 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
8271 Type *OpType = LHS->getType();
8272 CmpInst::Predicate Pred = I.getPredicate();
8273
8274 bool FloorX = match(LHS, m_Intrinsic<Intrinsic::floor>(m_Specific(RHS)));
8275 bool CeilX = match(LHS, m_Intrinsic<Intrinsic::ceil>(m_Specific(RHS)));
8276
8277 if (!FloorX && !CeilX) {
8278 if ((FloorX = match(RHS, m_Intrinsic<Intrinsic::floor>(m_Specific(LHS)))) ||
8279 (CeilX = match(RHS, m_Intrinsic<Intrinsic::ceil>(m_Specific(LHS))))) {
8280 std::swap(LHS, RHS);
8281 Pred = I.getSwappedPredicate();
8282 }
8283 }
8284
8285 switch (Pred) {
8286 case FCmpInst::FCMP_OLE:
8287 // fcmp ole floor(x), x => fcmp ord x, 0
8288 if (FloorX)
8290 "", &I);
8291 break;
8292 case FCmpInst::FCMP_OGT:
8293 // fcmp ogt floor(x), x => false
8294 if (FloorX)
8295 return IC.replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8296 break;
8297 case FCmpInst::FCMP_OGE:
8298 // fcmp oge ceil(x), x => fcmp ord x, 0
8299 if (CeilX)
8301 "", &I);
8302 break;
8303 case FCmpInst::FCMP_OLT:
8304 // fcmp olt ceil(x), x => false
8305 if (CeilX)
8306 return IC.replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8307 break;
8308 case FCmpInst::FCMP_ULE:
8309 // fcmp ule floor(x), x => true
8310 if (FloorX)
8311 return IC.replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8312 break;
8313 case FCmpInst::FCMP_UGT:
8314 // fcmp ugt floor(x), x => fcmp uno x, 0
8315 if (FloorX)
8317 "", &I);
8318 break;
8319 case FCmpInst::FCMP_UGE:
8320 // fcmp uge ceil(x), x => true
8321 if (CeilX)
8322 return IC.replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8323 break;
8324 case FCmpInst::FCMP_ULT:
8325 // fcmp ult ceil(x), x => fcmp uno x, 0
8326 if (CeilX)
8328 "", &I);
8329 break;
8330 default:
8331 break;
8332 }
8333
8334 return nullptr;
8335}
8336
8338 bool Changed = false;
8339
8340 /// Orders the operands of the compare so that they are listed from most
8341 /// complex to least complex. This puts constants before unary operators,
8342 /// before binary operators.
8343 if (getComplexity(I.getOperand(0)) < getComplexity(I.getOperand(1))) {
8344 I.swapOperands();
8345 Changed = true;
8346 }
8347
8348 const CmpInst::Predicate Pred = I.getPredicate();
8349 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
8350 if (Value *V = simplifyFCmpInst(Pred, Op0, Op1, I.getFastMathFlags(),
8352 return replaceInstUsesWith(I, V);
8353
8354 // Simplify 'fcmp pred X, X'
8355 Type *OpType = Op0->getType();
8356 assert(OpType == Op1->getType() && "fcmp with different-typed operands?");
8357 if (Op0 == Op1) {
8358 switch (Pred) {
8359 default: break;
8360 case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y)
8361 case FCmpInst::FCMP_ULT: // True if unordered or less than
8362 case FCmpInst::FCMP_UGT: // True if unordered or greater than
8363 case FCmpInst::FCMP_UNE: // True if unordered or not equal
8364 // Canonicalize these to be 'fcmp uno %X, 0.0'.
8365 I.setPredicate(FCmpInst::FCMP_UNO);
8366 I.setOperand(1, Constant::getNullValue(OpType));
8367 return &I;
8368
8369 case FCmpInst::FCMP_ORD: // True if ordered (no nans)
8370 case FCmpInst::FCMP_OEQ: // True if ordered and equal
8371 case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal
8372 case FCmpInst::FCMP_OLE: // True if ordered and less than or equal
8373 // Canonicalize these to be 'fcmp ord %X, 0.0'.
8374 I.setPredicate(FCmpInst::FCMP_ORD);
8375 I.setOperand(1, Constant::getNullValue(OpType));
8376 return &I;
8377 }
8378 }
8379
8380 if (I.isCommutative()) {
8381 if (auto Pair = matchSymmetricPair(I.getOperand(0), I.getOperand(1))) {
8382 replaceOperand(I, 0, Pair->first);
8383 replaceOperand(I, 1, Pair->second);
8384 return &I;
8385 }
8386 }
8387
8388 // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand,
8389 // then canonicalize the operand to 0.0.
8390 if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
8391 if (!match(Op0, m_PosZeroFP()) &&
8392 isKnownNeverNaN(Op0, 0, getSimplifyQuery().getWithInstruction(&I)))
8393 return replaceOperand(I, 0, ConstantFP::getZero(OpType));
8394
8395 if (!match(Op1, m_PosZeroFP()) &&
8396 isKnownNeverNaN(Op1, 0, getSimplifyQuery().getWithInstruction(&I)))
8397 return replaceOperand(I, 1, ConstantFP::getZero(OpType));
8398 }
8399
8400 // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y
8401 Value *X, *Y;
8402 if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y))))
8403 return new FCmpInst(I.getSwappedPredicate(), X, Y, "", &I);
8404
8406 return R;
8407
8408 // Test if the FCmpInst instruction is used exclusively by a select as
8409 // part of a minimum or maximum operation. If so, refrain from doing
8410 // any other folding. This helps out other analyses which understand
8411 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
8412 // and CodeGen. And in this case, at least one of the comparison
8413 // operands has at least one user besides the compare (the select),
8414 // which would often largely negate the benefit of folding anyway.
8415 if (I.hasOneUse())
8416 if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) {
8417 Value *A, *B;
8419 if (SPR.Flavor != SPF_UNKNOWN)
8420 return nullptr;
8421 }
8422
8423 // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0:
8424 // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0
8425 if (match(Op1, m_AnyZeroFP()) && !match(Op1, m_PosZeroFP()))
8426 return replaceOperand(I, 1, ConstantFP::getZero(OpType));
8427
8428 // Canonicalize:
8429 // fcmp olt X, +inf -> fcmp one X, +inf
8430 // fcmp ole X, +inf -> fcmp ord X, 0
8431 // fcmp ogt X, +inf -> false
8432 // fcmp oge X, +inf -> fcmp oeq X, +inf
8433 // fcmp ult X, +inf -> fcmp une X, +inf
8434 // fcmp ule X, +inf -> true
8435 // fcmp ugt X, +inf -> fcmp uno X, 0
8436 // fcmp uge X, +inf -> fcmp ueq X, +inf
8437 // fcmp olt X, -inf -> false
8438 // fcmp ole X, -inf -> fcmp oeq X, -inf
8439 // fcmp ogt X, -inf -> fcmp one X, -inf
8440 // fcmp oge X, -inf -> fcmp ord X, 0
8441 // fcmp ult X, -inf -> fcmp uno X, 0
8442 // fcmp ule X, -inf -> fcmp ueq X, -inf
8443 // fcmp ugt X, -inf -> fcmp une X, -inf
8444 // fcmp uge X, -inf -> true
8445 const APFloat *C;
8446 if (match(Op1, m_APFloat(C)) && C->isInfinity()) {
8447 switch (C->isNegative() ? FCmpInst::getSwappedPredicate(Pred) : Pred) {
8448 default:
8449 break;
8450 case FCmpInst::FCMP_ORD:
8451 case FCmpInst::FCMP_UNO:
8454 case FCmpInst::FCMP_OGT:
8455 case FCmpInst::FCMP_ULE:
8456 llvm_unreachable("Should be simplified by InstSimplify");
8457 case FCmpInst::FCMP_OLT:
8458 return new FCmpInst(FCmpInst::FCMP_ONE, Op0, Op1, "", &I);
8459 case FCmpInst::FCMP_OLE:
8460 return new FCmpInst(FCmpInst::FCMP_ORD, Op0, ConstantFP::getZero(OpType),
8461 "", &I);
8462 case FCmpInst::FCMP_OGE:
8463 return new FCmpInst(FCmpInst::FCMP_OEQ, Op0, Op1, "", &I);
8464 case FCmpInst::FCMP_ULT:
8465 return new FCmpInst(FCmpInst::FCMP_UNE, Op0, Op1, "", &I);
8466 case FCmpInst::FCMP_UGT:
8467 return new FCmpInst(FCmpInst::FCMP_UNO, Op0, ConstantFP::getZero(OpType),
8468 "", &I);
8469 case FCmpInst::FCMP_UGE:
8470 return new FCmpInst(FCmpInst::FCMP_UEQ, Op0, Op1, "", &I);
8471 }
8472 }
8473
8474 // Ignore signbit of bitcasted int when comparing equality to FP 0.0:
8475 // fcmp oeq/une (bitcast X), 0.0 --> (and X, SignMaskC) ==/!= 0
8476 if (match(Op1, m_PosZeroFP()) &&
8479 if (Pred == FCmpInst::FCMP_OEQ)
8480 IntPred = ICmpInst::ICMP_EQ;
8481 else if (Pred == FCmpInst::FCMP_UNE)
8482 IntPred = ICmpInst::ICMP_NE;
8483
8484 if (IntPred != ICmpInst::BAD_ICMP_PREDICATE) {
8485 Type *IntTy = X->getType();
8486 const APInt &SignMask = ~APInt::getSignMask(IntTy->getScalarSizeInBits());
8487 Value *MaskX = Builder.CreateAnd(X, ConstantInt::get(IntTy, SignMask));
8488 return new ICmpInst(IntPred, MaskX, ConstantInt::getNullValue(IntTy));
8489 }
8490 }
8491
8492 // Handle fcmp with instruction LHS and constant RHS.
8493 Instruction *LHSI;
8494 Constant *RHSC;
8495 if (match(Op0, m_Instruction(LHSI)) && match(Op1, m_Constant(RHSC))) {
8496 switch (LHSI->getOpcode()) {
8497 case Instruction::Select:
8498 // fcmp eq (cond ? x : -x), 0 --> fcmp eq x, 0
8499 if (FCmpInst::isEquality(Pred) && match(RHSC, m_AnyZeroFP()) &&
8501 return replaceOperand(I, 0, X);
8502 if (Instruction *NV = FoldOpIntoSelect(I, cast<SelectInst>(LHSI)))
8503 return NV;
8504 break;
8505 case Instruction::FSub:
8506 if (LHSI->hasOneUse())
8507 if (Instruction *NV = foldFCmpFSubIntoFCmp(I, LHSI, RHSC, *this))
8508 return NV;
8509 break;
8510 case Instruction::PHI:
8511 if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI)))
8512 return NV;
8513 break;
8514 case Instruction::SIToFP:
8515 case Instruction::UIToFP:
8516 if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC))
8517 return NV;
8518 break;
8519 case Instruction::FDiv:
8520 if (Instruction *NV = foldFCmpReciprocalAndZero(I, LHSI, RHSC))
8521 return NV;
8522 break;
8523 case Instruction::Load:
8524 if (auto *GEP = dyn_cast<GetElementPtrInst>(LHSI->getOperand(0)))
8525 if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)))
8527 cast<LoadInst>(LHSI), GEP, GV, I))
8528 return Res;
8529 break;
8530 }
8531 }
8532
8533 if (Instruction *R = foldFabsWithFcmpZero(I, *this))
8534 return R;
8535
8536 if (Instruction *R = foldSqrtWithFcmpZero(I, *this))
8537 return R;
8538
8539 if (Instruction *R = foldFCmpWithFloorAndCeil(I, *this))
8540 return R;
8541
8542 if (match(Op0, m_FNeg(m_Value(X)))) {
8543 // fcmp pred (fneg X), C --> fcmp swap(pred) X, -C
8544 Constant *C;
8545 if (match(Op1, m_Constant(C)))
8546 if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL))
8547 return new FCmpInst(I.getSwappedPredicate(), X, NegC, "", &I);
8548 }
8549
8550 // fcmp (fadd X, 0.0), Y --> fcmp X, Y
8551 if (match(Op0, m_FAdd(m_Value(X), m_AnyZeroFP())))
8552 return new FCmpInst(Pred, X, Op1, "", &I);
8553
8554 // fcmp X, (fadd Y, 0.0) --> fcmp X, Y
8555 if (match(Op1, m_FAdd(m_Value(Y), m_AnyZeroFP())))
8556 return new FCmpInst(Pred, Op0, Y, "", &I);
8557
8558 if (match(Op0, m_FPExt(m_Value(X)))) {
8559 // fcmp (fpext X), (fpext Y) -> fcmp X, Y
8560 if (match(Op1, m_FPExt(m_Value(Y))) && X->getType() == Y->getType())
8561 return new FCmpInst(Pred, X, Y, "", &I);
8562
8563 const APFloat *C;
8564 if (match(Op1, m_APFloat(C))) {
8565 const fltSemantics &FPSem =
8566 X->getType()->getScalarType()->getFltSemantics();
8567 bool Lossy;
8568 APFloat TruncC = *C;
8569 TruncC.convert(FPSem, APFloat::rmNearestTiesToEven, &Lossy);
8570
8571 if (Lossy) {
8572 // X can't possibly equal the higher-precision constant, so reduce any
8573 // equality comparison.
8574 // TODO: Other predicates can be handled via getFCmpCode().
8575 switch (Pred) {
8576 case FCmpInst::FCMP_OEQ:
8577 // X is ordered and equal to an impossible constant --> false
8578 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8579 case FCmpInst::FCMP_ONE:
8580 // X is ordered and not equal to an impossible constant --> ordered
8581 return new FCmpInst(FCmpInst::FCMP_ORD, X,
8582 ConstantFP::getZero(X->getType()));
8583 case FCmpInst::FCMP_UEQ:
8584 // X is unordered or equal to an impossible constant --> unordered
8585 return new FCmpInst(FCmpInst::FCMP_UNO, X,
8586 ConstantFP::getZero(X->getType()));
8587 case FCmpInst::FCMP_UNE:
8588 // X is unordered or not equal to an impossible constant --> true
8589 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8590 default:
8591 break;
8592 }
8593 }
8594
8595 // fcmp (fpext X), C -> fcmp X, (fptrunc C) if fptrunc is lossless
8596 // Avoid lossy conversions and denormals.
8597 // Zero is a special case that's OK to convert.
8598 APFloat Fabs = TruncC;
8599 Fabs.clearSign();
8600 if (!Lossy &&
8601 (Fabs.isZero() || !(Fabs < APFloat::getSmallestNormalized(FPSem)))) {
8602 Constant *NewC = ConstantFP::get(X->getType(), TruncC);
8603 return new FCmpInst(Pred, X, NewC, "", &I);
8604 }
8605 }
8606 }
8607
8608 // Convert a sign-bit test of an FP value into a cast and integer compare.
8609 // TODO: Simplify if the copysign constant is 0.0 or NaN.
8610 // TODO: Handle non-zero compare constants.
8611 // TODO: Handle other predicates.
8612 if (match(Op0, m_OneUse(m_Intrinsic<Intrinsic::copysign>(m_APFloat(C),
8613 m_Value(X)))) &&
8614 match(Op1, m_AnyZeroFP()) && !C->isZero() && !C->isNaN()) {
8615 Type *IntType = Builder.getIntNTy(X->getType()->getScalarSizeInBits());
8616 if (auto *VecTy = dyn_cast<VectorType>(OpType))
8617 IntType = VectorType::get(IntType, VecTy->getElementCount());
8618
8619 // copysign(non-zero constant, X) < 0.0 --> (bitcast X) < 0
8620 if (Pred == FCmpInst::FCMP_OLT) {
8621 Value *IntX = Builder.CreateBitCast(X, IntType);
8622 return new ICmpInst(ICmpInst::ICMP_SLT, IntX,
8623 ConstantInt::getNullValue(IntType));
8624 }
8625 }
8626
8627 {
8628 Value *CanonLHS = nullptr, *CanonRHS = nullptr;
8629 match(Op0, m_Intrinsic<Intrinsic::canonicalize>(m_Value(CanonLHS)));
8630 match(Op1, m_Intrinsic<Intrinsic::canonicalize>(m_Value(CanonRHS)));
8631
8632 // (canonicalize(x) == x) => (x == x)
8633 if (CanonLHS == Op1)
8634 return new FCmpInst(Pred, Op1, Op1, "", &I);
8635
8636 // (x == canonicalize(x)) => (x == x)
8637 if (CanonRHS == Op0)
8638 return new FCmpInst(Pred, Op0, Op0, "", &I);
8639
8640 // (canonicalize(x) == canonicalize(y)) => (x == y)
8641 if (CanonLHS && CanonRHS)
8642 return new FCmpInst(Pred, CanonLHS, CanonRHS, "", &I);
8643 }
8644
8645 if (I.getType()->isVectorTy())
8646 if (Instruction *Res = foldVectorCmp(I, Builder))
8647 return Res;
8648
8649 return Changed ? &I : nullptr;
8650}
AMDGPU Register Bank Select
Rewrite undef for PHI
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
#define Check(C,...)
Hexagon Common GEP
static Instruction * foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI, Constant *RHSC)
Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary.
static Instruction * foldFabsWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC)
Optimize fabs(X) compared with zero.
static Value * rewriteGEPAsOffset(Value *Start, Value *Base, GEPNoWrapFlags NW, const DataLayout &DL, SetVector< Value * > &Explored, InstCombiner &IC)
Returns a re-written value of Start as an indexed GEP using Base as a pointer.
static bool addWithOverflow(APInt &Result, const APInt &In1, const APInt &In2, bool IsSigned=false)
Compute Result = In1+In2, returning true if the result overflowed for this type.
static Instruction * foldICmpAndXX(ICmpInst &I, const SimplifyQuery &Q, InstCombinerImpl &IC)
static Instruction * foldVectorCmp(CmpInst &Cmp, InstCombiner::BuilderTy &Builder)
static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q, unsigned Depth=0)
static Value * createLogicFromTable(const std::bitset< 4 > &Table, Value *Op0, Value *Op1, IRBuilderBase &Builder, bool HasOneUse)
static Instruction * foldICmpOfUAddOv(ICmpInst &I)
static bool isChainSelectCmpBranch(const SelectInst *SI)
Return true when the instruction sequence within a block is select-cmp-br.
static Instruction * foldICmpInvariantGroup(ICmpInst &I)
static Instruction * foldReductionIdiom(ICmpInst &I, InstCombiner::BuilderTy &Builder, const DataLayout &DL)
This function folds patterns produced by lowering of reduce idioms, such as llvm.vector....
static Instruction * canonicalizeICmpBool(ICmpInst &I, InstCombiner::BuilderTy &Builder)
Integer compare with boolean values can always be turned into bitwise ops.
static Instruction * foldFCmpFSubIntoFCmp(FCmpInst &I, Instruction *LHSI, Constant *RHSC, InstCombinerImpl &CI)
static Value * foldICmpOrXorSubChain(ICmpInst &Cmp, BinaryOperator *Or, InstCombiner::BuilderTy &Builder)
Fold icmp eq/ne (or (xor/sub (X1, X2), xor/sub (X3, X4))), 0.
static bool hasBranchUse(ICmpInst &I)
Given an icmp instruction, return true if any use of this comparison is a branch on sign bit comparis...
static Value * foldICmpWithLowBitMaskedVal(CmpPredicate Pred, Value *Op0, Value *Op1, const SimplifyQuery &Q, InstCombiner &IC)
Some comparisons can be simplified.
static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth)
When performing a comparison against a constant, it is possible that not all the bits in the LHS are ...
static Instruction * foldICmpShlLHSC(ICmpInst &Cmp, Instruction *Shl, const APInt &C)
Fold icmp (shl nuw C2, Y), C.
static Instruction * foldFCmpWithFloorAndCeil(FCmpInst &I, InstCombinerImpl &IC)
static Instruction * foldICmpXorXX(ICmpInst &I, const SimplifyQuery &Q, InstCombinerImpl &IC)
static Instruction * foldICmpOfCmpIntrinsicWithConstant(CmpPredicate Pred, IntrinsicInst *I, const APInt &C, InstCombiner::BuilderTy &Builder)
static Instruction * processUMulZExtIdiom(ICmpInst &I, Value *MulVal, const APInt *OtherVal, InstCombinerImpl &IC)
Recognize and process idiom involving test for multiplication overflow.
static Instruction * foldSqrtWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC)
Optimize sqrt(X) compared with zero.
static Instruction * foldFCmpFNegCommonOp(FCmpInst &I)
static Instruction * foldICmpWithHighBitMask(ICmpInst &Cmp, InstCombiner::BuilderTy &Builder)
static ICmpInst * canonicalizeCmpWithConstant(ICmpInst &I)
If we have an icmp le or icmp ge instruction with a constant operand, turn it into the appropriate ic...
static Instruction * foldICmpIntrinsicWithIntrinsic(ICmpInst &Cmp, InstCombiner::BuilderTy &Builder)
Fold an icmp with LLVM intrinsics.
static Instruction * foldICmpUSubSatOrUAddSatWithConstant(CmpPredicate Pred, SaturatingInst *II, const APInt &C, InstCombiner::BuilderTy &Builder)
static Instruction * foldICmpPow2Test(ICmpInst &I, InstCombiner::BuilderTy &Builder)
static bool subWithOverflow(APInt &Result, const APInt &In1, const APInt &In2, bool IsSigned=false)
Compute Result = In1-In2, returning true if the result overflowed for this type.
static bool canRewriteGEPAsOffset(Value *Start, Value *Base, GEPNoWrapFlags &NW, const DataLayout &DL, SetVector< Value * > &Explored)
Returns true if we can rewrite Start as a GEP with pointer Base and some integer offset.
static Instruction * foldICmpXNegX(ICmpInst &I, InstCombiner::BuilderTy &Builder)
static Instruction * processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B, ConstantInt *CI2, ConstantInt *CI1, InstCombinerImpl &IC)
The caller has matched a pattern of the form: I = icmp ugt (add (add A, B), CI2), CI1 If this is of t...
static Value * foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ, InstCombiner::BuilderTy &Builder)
static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C)
Returns true if the exploded icmp can be expressed as a signed comparison to zero and updates the pre...
static Instruction * transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS, CmpPredicate Cond, const DataLayout &DL, InstCombiner &IC)
Converts (CMP GEPLHS, RHS) if this change would make RHS a constant.
static Instruction * foldCtpopPow2Test(ICmpInst &I, IntrinsicInst *CtpopLhs, const APInt &CRhs, InstCombiner::BuilderTy &Builder, const SimplifyQuery &Q)
static void setInsertionPoint(IRBuilder<> &Builder, Value *V, bool Before=true)
static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS, bool IsSigned)
static Value * foldICmpWithTruncSignExtendedVal(ICmpInst &I, InstCombiner::BuilderTy &Builder)
Some comparisons can be simplified.
static Instruction * foldICmpOrXX(ICmpInst &I, const SimplifyQuery &Q, InstCombinerImpl &IC)
This file provides internal interfaces used to implement the InstCombine.
This file provides the interface for the instcombine pass implementation.
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:533
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
mir Rename Register Operands
#define T1
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:166
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:39
Value * RHS
Value * LHS
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition: APFloat.cpp:5465
void clearSign()
Definition: APFloat.h:1295
bool isZero() const
Definition: APFloat.h:1436
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
Definition: APFloat.h:1155
APInt bitcastToAPInt() const
Definition: APFloat.h:1346
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition: APFloat.h:1135
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition: APFloat.h:1095
FPClassTest classify() const
Return the FPClassTest which will return true for the value.
Definition: APFloat.cpp:5452
opStatus roundToIntegral(roundingMode RM)
Definition: APFloat.h:1245
Class for arbitrary precision integers.
Definition: APInt.h:78
APInt udiv(const APInt &RHS) const
Unsigned division operation.
Definition: APInt.cpp:1547
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:234
static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
Definition: APInt.cpp:1732
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
Definition: APInt.h:449
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:986
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:229
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition: APInt.h:423
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1520
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1492
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:910
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:206
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition: APInt.h:1330
APInt abs() const
Get the absolute value.
Definition: APInt.h:1773
unsigned ceilLogBase2() const
Definition: APInt.h:1742
bool sgt(const APInt &RHS) const
Signed greater than comparison.
Definition: APInt.h:1201
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition: APInt.h:371
APInt usub_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1922
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition: APInt.h:1182
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition: APInt.h:380
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
Definition: APInt.h:466
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1468
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.h:1111
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:209
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
Definition: APInt.h:216
bool isNegative() const
Determine sign of this APInt.
Definition: APInt.h:329
APInt sadd_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1902
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1249
bool eq(const APInt &RHS) const
Equality comparison.
Definition: APInt.h:1079
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
Definition: APInt.cpp:1618
APInt uadd_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1909
void negate()
Negate this APInt in place.
Definition: APInt.h:1450
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition: APInt.h:1618
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition: APInt.h:1577
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:219
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Definition: APInt.h:356
unsigned countl_one() const
Count the number of leading one bits.
Definition: APInt.h:1594
unsigned logBase2() const
Definition: APInt.h:1739
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition: APInt.h:475
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition: APInt.h:827
bool isMask(unsigned numBits) const
Definition: APInt.h:488
bool isMaxSignedValue() const
Determine if this is the largest signed value.
Definition: APInt.h:405
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition: APInt.h:1150
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition: APInt.h:873
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:440
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:306
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition: APInt.h:296
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition: APInt.h:200
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
Definition: APInt.h:1237
APInt ssub_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1915
bool isOne() const
Determine if this is a value of 1.
Definition: APInt.h:389
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition: APInt.h:286
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:239
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:851
unsigned countr_one() const
Count the number of trailing one bits.
Definition: APInt.h:1635
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition: APInt.h:1221
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
an instruction to allocate memory on the stack
Definition: Instructions.h:63
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
Class to represent array types.
Definition: DerivedTypes.h:395
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:416
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:459
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:239
BinaryOps getOpcode() const
Definition: InstrTypes.h:370
static BinaryOperator * CreateNot(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
Conditional or Unconditional Branch instruction.
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1294
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:661
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Definition: InstrTypes.h:988
Predicate getStrictPredicate() const
For example, SGE -> SGT, SLE -> SLT, ULE -> ULT, UGE -> UGT.
Definition: InstrTypes.h:856
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:673
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:676
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:690
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:702
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:703
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:679
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:688
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:677
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:678
@ ICMP_UGE
unsigned greater or equal
Definition: InstrTypes.h:697
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:696
@ ICMP_SGT
signed greater than
Definition: InstrTypes.h:700
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:687
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:681
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:684
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:698
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:685
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:680
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:682
@ ICMP_EQ
equal
Definition: InstrTypes.h:694
@ ICMP_NE
not equal
Definition: InstrTypes.h:695
@ ICMP_SGE
signed greater or equal
Definition: InstrTypes.h:701
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:689
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:699
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:686
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:675
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:683
bool isSigned() const
Definition: InstrTypes.h:928
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition: InstrTypes.h:825
bool isTrueWhenEqual() const
This is just a convenience.
Definition: InstrTypes.h:940
static CmpInst * Create(OtherOps Op, Predicate Pred, Value *S1, Value *S2, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Construct a compare instruction, given the opcode, the predicate and the two operands.
Predicate getNonStrictPredicate() const
For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
Definition: InstrTypes.h:869
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition: InstrTypes.h:787
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:763
bool isStrictPredicate() const
Definition: InstrTypes.h:841
Predicate getFlippedStrictnessPredicate() const
For predicate of kind "is X or equal to 0" returns the predicate "is X".
Definition: InstrTypes.h:891
bool isIntPredicate() const
Definition: InstrTypes.h:781
bool isUnsigned() const
Definition: InstrTypes.h:934
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
Definition: CmpPredicate.h:22
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2307
static Constant * getPointerBitCastOrAddrSpaceCast(Constant *C, Type *Ty)
Create a BitCast or AddrSpaceCast for a pointer type depending on the address space.
Definition: Constants.cpp:2268
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2644
static Constant * getNot(Constant *C)
Definition: Constants.cpp:2631
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2293
static Constant * getXor(Constant *C1, Constant *C2)
Definition: Constants.cpp:2658
static Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2637
static Constant * getNeg(Constant *C, bool HasNSW=false)
Definition: Constants.cpp:2625
static Constant * getZero(Type *Ty, bool Negative=false)
Definition: Constants.cpp:1057
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
Definition: Constants.h:258
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:866
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:208
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
Definition: Constants.h:126
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:873
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
Definition: Constants.h:151
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:148
static ConstantInt * getBool(LLVMContext &Context, bool V)
Definition: Constants.cpp:880
This class represents a range of values.
Definition: ConstantRange.h:47
ConstantRange add(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an addition of a value in this ran...
std::optional< ConstantRange > exactUnionWith(const ConstantRange &CR) const
Union the two ranges and return the result if it can be represented exactly, otherwise return std::nu...
bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const
Set up Pred and RHS such that ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.
ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
const APInt * getSingleElement() const
If this set contains a single element, return it, otherwise return null.
ConstantRange truncate(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly smaller than the current typ...
ConstantRange difference(const ConstantRange &CR) const
Subtract the specified range from this range (aka relative complement of the sets).
bool isEmptySet() const
Return true if this set contains no members.
static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
ConstantRange inverse() const
Return a new range that is the logical not of the current set.
std::optional< ConstantRange > exactIntersectWith(const ConstantRange &CR) const
Intersect the two ranges and return the result if it can be represented exactly, otherwise return std...
ConstantRange intersectWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the intersection of this range with another range.
ConstantRange sub(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a subtraction of a value in this r...
static ConstantRange makeExactNoWrapRegion(Instruction::BinaryOps BinOp, const APInt &Other, unsigned NoWrapKind)
Produce the range that contains X if and only if "X BinOp Other" does not wrap.
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
Definition: Constants.cpp:1472
This is an important base class in LLVM.
Definition: Constant.h:42
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
Definition: Constants.cpp:403
static Constant * replaceUndefsWith(Constant *C, Constant *Replacement)
Try to replace undefined constant C or undefined elements in C with Replacement.
Definition: Constants.cpp:784
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:420
const APInt & getUniqueInteger() const
If C is a constant integer then return its value, otherwise C must be a vector of constant integers,...
Definition: Constants.cpp:1771
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:373
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition: Constants.cpp:90
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Definition: DataLayout.cpp:851
unsigned getPointerTypeSizeInBits(Type *) const
Layout pointer size, in bits, based on the type.
Definition: DataLayout.cpp:743
IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
Definition: DataLayout.cpp:878
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:457
Type * getSmallestLegalIntType(LLVMContext &C, unsigned Width=0) const
Returns the smallest integer type with size at least as big as Width bits.
Definition: DataLayout.cpp:866
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:156
iterator end()
Definition: DenseMap.h:84
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition: DenseMap.h:147
ArrayRef< BranchInst * > conditionsFor(const Value *V) const
Access the list of branches which affect this value.
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Definition: Dominators.cpp:122
static ExtractValueInst * Create(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This instruction compares its operands according to the predicate given to the constructor.
bool isEquality() const
Represents flags for the getelementptr instruction/expression.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
GEPNoWrapFlags intersectForOffsetAdd(GEPNoWrapFlags Other) const
Given (gep (gep p, x), y), determine the nowrap flags for (gep p, x+y).
static GEPNoWrapFlags none()
bool isInBounds() const
Test whether this is an inbounds GEP, as defined by LangRef.html.
Definition: Operator.h:435
Type * getSourceElementType() const
Definition: Operator.cpp:70
Value * getPointerOperand()
Definition: Operator.h:462
GEPNoWrapFlags getNoWrapFlags() const
Definition: Operator.h:430
bool hasAllConstantIndices() const
Return true if all of the indices of this GEP are constant integers.
Definition: Operator.h:509
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:933
Type * getValueType() const
Definition: GlobalValue.h:296
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool isConstant() const
If the value is a global constant, its value is immutable throughout the runtime execution of the pro...
bool hasDefinitiveInitializer() const
hasDefinitiveInitializer - Whether the global variable has an initializer, and any other instances of...
This instruction compares its operands according to the predicate given to the constructor.
static bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
Predicate getFlippedSignednessPredicate() const
For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->EQ.
Predicate getSignedPredicate() const
For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
bool isEquality() const
Return true if this predicate is either EQ or NE.
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
bool isRelational() const
Return true if the predicate is relational (not EQ or NE).
Predicate getUnsignedPredicate() const
For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:91
CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
Definition: IRBuilder.cpp:871
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2289
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2491
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Definition: IRBuilder.h:536
Value * CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2297
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1152
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition: IRBuilder.h:2547
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition: IRBuilder.h:463
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Definition: IRBuilder.cpp:890
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1460
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2277
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Definition: IRBuilder.h:1889
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
Definition: IRBuilder.h:1744
Value * createIsFPClass(Value *FPNum, unsigned Test)
Definition: IRBuilder.cpp:1237
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:483
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2398
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2429
Value * CreateNot(Value *V, const Twine &Name="")
Definition: IRBuilder.h:1772
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2273
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1367
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2155
Value * CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2281
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1439
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Definition: IRBuilder.h:2048
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1498
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2189
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1350
ConstantInt * getFalse()
Get the constant value for i1 false.
Definition: IRBuilder.h:468
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2444
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Definition: IRBuilder.h:2034
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1520
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1689
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2305
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:2227
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
Definition: IRBuilder.h:2575
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition: IRBuilder.h:177
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1542
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2383
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Definition: IRBuilder.h:513
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
Definition: IRBuilder.h:499
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1427
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1384
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2697
Instruction * foldICmpShrConstant(ICmpInst &Cmp, BinaryOperator *Shr, const APInt &C)
Fold icmp ({al}shr X, Y), C.
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false)
Given an instruction with a select as one operand and a constant as the other operand,...
Instruction * foldICmpWithZextOrSext(ICmpInst &ICmp)
Instruction * foldICmpSelectConstant(ICmpInst &Cmp, SelectInst *Select, ConstantInt *C)
Instruction * foldICmpSRemConstant(ICmpInst &Cmp, BinaryOperator *UDiv, const APInt &C)
Instruction * foldICmpBinOpWithConstant(ICmpInst &Cmp, BinaryOperator *BO, const APInt &C)
Fold an icmp with BinaryOp and constant operand: icmp Pred BO, C.
Instruction * foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or, const APInt &C)
Fold icmp (or X, Y), C.
Instruction * foldICmpTruncWithTruncOrExt(ICmpInst &Cmp, const SimplifyQuery &Q)
Fold icmp (trunc nuw/nsw X), (trunc nuw/nsw Y).
Instruction * foldSignBitTest(ICmpInst &I)
Fold equality-comparison between zero and any (maybe truncated) right-shift by one-less-than-bitwidth...
Instruction * foldOpIntoPhi(Instruction &I, PHINode *PN, bool AllowMultipleUses=false)
Given a binary operator, cast instruction, or select which has a PHI node as operand #0,...
Value * insertRangeTest(Value *V, const APInt &Lo, const APInt &Hi, bool isSigned, bool Inside)
Emit a computation of: (V >= Lo && V < Hi) if Inside is true, otherwise (V < Lo || V >= Hi).
Instruction * foldICmpBinOp(ICmpInst &Cmp, const SimplifyQuery &SQ)
Try to fold icmp (binop), X or icmp X, (binop).
Instruction * foldICmpSubConstant(ICmpInst &Cmp, BinaryOperator *Sub, const APInt &C)
Fold icmp (sub X, Y), C.
Instruction * foldICmpInstWithConstantNotInt(ICmpInst &Cmp)
Handle icmp with constant (but not simple integer constant) RHS.
Instruction * foldICmpShlConstConst(ICmpInst &I, Value *ShAmt, const APInt &C1, const APInt &C2)
Handle "(icmp eq/ne (shl AP2, A), AP1)" -> (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)).
Value * reassociateShiftAmtsOfTwoSameDirectionShifts(BinaryOperator *Sh0, const SimplifyQuery &SQ, bool AnalyzeForSignBitExtraction=false)
Instruction * foldICmpEqIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II, const APInt &C)
Fold an equality icmp with LLVM intrinsic and constant operand.
Value * foldMultiplicationOverflowCheck(ICmpInst &Cmp)
Fold (-1 u/ x) u< y ((x * y) ?/ x) != y to @llvm.
Instruction * foldICmpWithConstant(ICmpInst &Cmp)
Fold icmp Pred X, C.
CmpInst * canonicalizeICmpPredicate(CmpInst &I)
If we have a comparison with a non-canonical predicate, if we can update all the users,...
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Instruction * foldICmpWithZero(ICmpInst &Cmp)
Instruction * foldICmpCommutative(CmpPredicate Pred, Value *Op0, Value *Op1, ICmpInst &CxtI)
Instruction * foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp, BinaryOperator *BO, const APInt &C)
Fold an icmp equality instruction with binary operator LHS and constant RHS: icmp eq/ne BO,...
Instruction * foldICmpUsingBoolRange(ICmpInst &I)
If one operand of an icmp is effectively a bool (value range of {0,1}), then try to reduce patterns b...
Instruction * foldICmpWithTrunc(ICmpInst &Cmp)
Instruction * foldICmpIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II, const APInt &C)
Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
bool SimplifyDemandedBits(Instruction *I, unsigned Op, const APInt &DemandedMask, KnownBits &Known, unsigned Depth, const SimplifyQuery &Q) override
This form of SimplifyDemandedBits simplifies the specified instruction operand if possible,...
bool matchThreeWayIntCompare(SelectInst *SI, Value *&LHS, Value *&RHS, ConstantInt *&Less, ConstantInt *&Equal, ConstantInt *&Greater)
Match a select chain which produces one of three values based on whether the LHS is less than,...
Instruction * foldCmpLoadFromIndexedGlobal(LoadInst *LI, GetElementPtrInst *GEP, GlobalVariable *GV, CmpInst &ICI, ConstantInt *AndCst=nullptr)
This is called when we see this pattern: cmp pred (load (gep GV, ...)), cmpcst where GV is a global v...
Instruction * visitFCmpInst(FCmpInst &I)
Instruction * foldICmpUsingKnownBits(ICmpInst &Cmp)
Try to fold the comparison based on range information we can get by checking whether bits are known t...
Instruction * foldICmpDivConstant(ICmpInst &Cmp, BinaryOperator *Div, const APInt &C)
Fold icmp ({su}div X, Y), C.
Instruction * foldIRemByPowerOfTwoToBitTest(ICmpInst &I)
If we have: icmp eq/ne (urem/srem x, y), 0 iff y is a power-of-two, we can replace this with a bit te...
Instruction * foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, Constant *RHSC)
Fold fcmp ([us]itofp x, cst) if possible.
Instruction * foldICmpUDivConstant(ICmpInst &Cmp, BinaryOperator *UDiv, const APInt &C)
Fold icmp (udiv X, Y), C.
Instruction * foldICmpAddOpConst(Value *X, const APInt &C, CmpPredicate Pred)
Fold "icmp pred (X+C), X".
Constant * getLosslessTrunc(Constant *C, Type *TruncTy, unsigned ExtOp)
Instruction * foldICmpWithCastOp(ICmpInst &ICmp)
Handle icmp (cast x), (cast or constant).
Instruction * foldICmpTruncConstant(ICmpInst &Cmp, TruncInst *Trunc, const APInt &C)
Fold icmp (trunc X), C.
Instruction * foldICmpAddConstant(ICmpInst &Cmp, BinaryOperator *Add, const APInt &C)
Fold icmp (add X, Y), C.
Instruction * foldICmpMulConstant(ICmpInst &Cmp, BinaryOperator *Mul, const APInt &C)
Fold icmp (mul X, Y), C.
Instruction * tryFoldInstWithCtpopWithNot(Instruction *I)
Instruction * foldICmpXorConstant(ICmpInst &Cmp, BinaryOperator *Xor, const APInt &C)
Fold icmp (xor X, Y), C.
Instruction * foldSelectICmp(CmpPredicate Pred, SelectInst *SI, Value *RHS, const ICmpInst &I)
Instruction * foldICmpInstWithConstantAllowPoison(ICmpInst &Cmp, const APInt &C)
Try to fold integer comparisons with a constant operand: icmp Pred X, C where X is some kind of instr...
Instruction * foldICmpAndShift(ICmpInst &Cmp, BinaryOperator *And, const APInt &C1, const APInt &C2)
Fold icmp (and (sh X, Y), C2), C1.
Instruction * foldICmpInstWithConstant(ICmpInst &Cmp)
Try to fold integer comparisons with a constant operand: icmp Pred X, C where X is some kind of instr...
Instruction * foldICmpXorShiftConst(ICmpInst &Cmp, BinaryOperator *Xor, const APInt &C)
For power-of-2 C: ((X s>> ShiftC) ^ X) u< C --> (X + C) u< (C << 1) ((X s>> ShiftC) ^ X) u> (C - 1) -...
Instruction * foldICmpShlConstant(ICmpInst &Cmp, BinaryOperator *Shl, const APInt &C)
Fold icmp (shl X, Y), C.
Instruction * foldICmpAndConstant(ICmpInst &Cmp, BinaryOperator *And, const APInt &C)
Fold icmp (and X, Y), C.
Instruction * foldICmpEquality(ICmpInst &Cmp)
Instruction * foldICmpWithMinMax(Instruction &I, MinMaxIntrinsic *MinMax, Value *Z, CmpPredicate Pred)
Fold icmp Pred min|max(X, Y), Z.
bool dominatesAllUses(const Instruction *DI, const Instruction *UI, const BasicBlock *DB) const
True when DB dominates all uses of DI except UI.
bool foldAllocaCmp(AllocaInst *Alloca)
Instruction * visitICmpInst(ICmpInst &I)
OverflowResult computeOverflow(Instruction::BinaryOps BinaryOp, bool IsSigned, Value *LHS, Value *RHS, Instruction *CxtI) const
Instruction * foldICmpWithDominatingICmp(ICmpInst &Cmp)
Canonicalize icmp instructions based on dominating conditions.
bool replacedSelectWithOperand(SelectInst *SI, const ICmpInst *Icmp, const unsigned SIOpd)
Try to replace select with select operand SIOpd in SI-ICmp sequence.
Instruction * foldICmpShrConstConst(ICmpInst &I, Value *ShAmt, const APInt &C1, const APInt &C2)
Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" -> (icmp eq/ne A, Log2(AP2/AP1)) -> (icmp eq/ne A,...
void freelyInvertAllUsersOf(Value *V, Value *IgnoredUser=nullptr)
Freely adapt every user of V as-if V was changed to !V.
Instruction * foldICmpAndConstConst(ICmpInst &Cmp, BinaryOperator *And, const APInt &C1)
Fold icmp (and X, C2), C1.
Instruction * foldICmpBitCast(ICmpInst &Cmp)
Instruction * foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, CmpPredicate Cond, Instruction &I)
Fold comparisons between a GEP instruction and something else.
The core instruction combiner logic.
Definition: InstCombiner.h:48
OverflowResult computeOverflowForSignedSub(const Value *LHS, const Value *RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:508
SimplifyQuery SQ
Definition: InstCombiner.h:77
bool isFreeToInvert(Value *V, bool WillInvertAllUses, bool &DoesConsume)
Return true if the specified value is free to invert (apply ~ to).
Definition: InstCombiner.h:234
OverflowResult computeOverflowForUnsignedMul(const Value *LHS, const Value *RHS, const Instruction *CxtI, bool IsNSW=false) const
Definition: InstCombiner.h:471
static unsigned getComplexity(Value *V)
Assign a complexity or rank value to LLVM Values.
Definition: InstCombiner.h:143
TargetLibraryInfo & TLI
Definition: InstCombiner.h:74
bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero=false, unsigned Depth=0, const Instruction *CxtI=nullptr)
Definition: InstCombiner.h:449
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
Definition: InstCombiner.h:394
uint64_t MaxArraySizeForCombine
Maximum size of array considered when transforming.
Definition: InstCombiner.h:56
OverflowResult computeOverflowForSignedAdd(const WithCache< const Value * > &LHS, const WithCache< const Value * > &RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:494
static Constant * SubOne(Constant *C)
Subtract one from a Constant.
Definition: InstCombiner.h:183
static std::optional< std::pair< CmpPredicate, Constant * > > getFlippedStrictnessPredicateAndConstant(CmpPredicate Pred, Constant *C)
OverflowResult computeOverflowForUnsignedSub(const Value *LHS, const Value *RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:501
static bool isCanonicalPredicate(CmpPredicate Pred)
Predicate canonicalization reduces the number of patterns that need to be matched by other transforms...
Definition: InstCombiner.h:160
const DataLayout & DL
Definition: InstCombiner.h:76
DomConditionCache DC
Definition: InstCombiner.h:82
bool canFreelyInvertAllUsersOf(Instruction *V, Value *IgnoredUser)
Given i1 V, can every user of V be freely adapted if V is changed to !V ? InstCombine's freelyInvertA...
Definition: InstCombiner.h:250
AssumptionCache & AC
Definition: InstCombiner.h:73
void addToWorklist(Instruction *I)
Definition: InstCombiner.h:338
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
Definition: InstCombiner.h:418
DominatorTree & DT
Definition: InstCombiner.h:75
OverflowResult computeOverflowForSignedMul(const Value *LHS, const Value *RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:479
void computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth, const Instruction *CxtI) const
Definition: InstCombiner.h:439
BuilderTy & Builder
Definition: InstCombiner.h:61
OverflowResult computeOverflowForUnsignedAdd(const WithCache< const Value * > &LHS, const WithCache< const Value * > &RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:486
Value * getFreelyInverted(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume)
Definition: InstCombiner.h:215
const SimplifyQuery & getSimplifyQuery() const
Definition: InstCombiner.h:344
unsigned ComputeMaxSignificantBits(const Value *Op, unsigned Depth=0, const Instruction *CxtI=nullptr) const
Definition: InstCombiner.h:466
bool hasNoNaNs() const LLVM_READONLY
Determine whether the no-NaNs flag is set.
bool hasNoUnsignedWrap() const LLVM_READONLY
Determine whether the no unsigned wrap flag is set.
bool hasNoInfs() const LLVM_READONLY
Determine whether the no-infs flag is set.
bool isArithmeticShift() const
Return true if this is an arithmetic shift right.
Definition: Instruction.h:323
bool hasNoSignedWrap() const LLVM_READONLY
Determine whether the no signed wrap flag is set.
bool isCommutative() const LLVM_READONLY
Return true if the instruction is commutative:
bool isExact() const LLVM_READONLY
Determine whether the exact flag is set.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:274
bool isShift() const
Definition: Instruction.h:282
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:311
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:48
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:55
An instruction for reading from memory.
Definition: Instructions.h:176
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:205
This class represents min/max intrinsics.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Represents a saturating add/sub intrinsic.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, Instruction *MDFrom=nullptr)
A vector that has set insertion semantics.
Definition: SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
Definition: SetVector.h:254
This instruction constructs a fixed permutation of two input vectors.
bool empty() const
Definition: SmallVector.h:81
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:937
void push_back(const T &Elt)
Definition: SmallVector.h:413
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
Class to represent struct types.
Definition: DerivedTypes.h:218
This class represents a truncation of integer types.
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property.
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
unsigned getIntegerBitWidth() const
const fltSemantics & getFltSemantics() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:270
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition: Type.h:243
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:264
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
Definition: Type.h:165
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
int getFPMantissaWidth() const
Return the width of the mantissa of this type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:237
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:355
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
void setOperand(unsigned i, Value *Val)
Definition: User.h:233
Value * getOperand(unsigned i) const
Definition: User.h:228
unsigned getNumOperands() const
Definition: User.h:250
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr) const
Accumulate the constant offset this value has compared to a base pointer.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
iterator_range< user_iterator > users()
Definition: Value.h:421
bool hasNUsesOrMore(unsigned N) const
Return true if this value has N uses or more.
Definition: Value.cpp:153
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:694
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
iterator_range< use_iterator > uses()
Definition: Value.h:376
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:383
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:202
const ParentTy * getParent() const
Definition: ilist_node.h:32
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt RoundingUDiv(const APInt &A, const APInt &B, APInt::Rounding RM)
Return A unsign-divided by B, rounded by the given rounding mode.
Definition: APInt.cpp:2736
APInt RoundingSDiv(const APInt &A, const APInt &B, APInt::Rounding RM)
Return A sign-divided by B, rounded by the given rounding mode.
Definition: APInt.cpp:2754
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition: Intrinsics.cpp:731
cst_pred_ty< is_all_ones > m_AllOnes()
Match an integer or vector with all bits set.
Definition: PatternMatch.h:524
cst_pred_ty< is_lowbit_mask > m_LowBitMask()
Match an integer or vector with only the low bit(s) set.
Definition: PatternMatch.h:673
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
Definition: PatternMatch.h:100
cst_pred_ty< is_sign_mask > m_SignMask()
Match an integer or vector with only the sign bit(s) set.
Definition: PatternMatch.h:664
OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
cst_pred_ty< is_power2 > m_Power2()
Match an integer or vector power-of-2.
Definition: PatternMatch.h:619
BinaryOp_match< LHS, RHS, Instruction::URem > m_URem(const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, TruncInst >, OpTy > m_TruncOrSelf(const OpTy &Op)
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
Definition: PatternMatch.h:165
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
Definition: PatternMatch.h:982
match_combine_or< CastInst_match< OpTy, ZExtInst >, OpTy > m_ZExtOrSelf(const OpTy &Op)
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
BinOpPred_match< LHS, RHS, is_idiv_op > m_IDiv(const LHS &L, const RHS &R)
Matches integer division operations.
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
Definition: PatternMatch.h:826
cstfp_pred_ty< is_any_zero_fp > m_AnyZeroFP()
Match a floating-point negative zero or positive zero.
Definition: PatternMatch.h:764
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:885
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
specific_intval< true > m_SpecificIntAllowPoison(const APInt &V)
Definition: PatternMatch.h:990
CmpClass_match< LHS, RHS, ICmpInst, true > m_c_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
Matches an ICmp with a predicate over LHS and RHS in either order.
OverflowingBinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub, OverflowingBinaryOperator::NoSignedWrap > m_NSWNeg(const ValTy &V)
Matches a 'Neg' as 'sub nsw 0, V'.
cst_pred_ty< is_nonnegative > m_NonNegative()
Match an integer or vector of non-negative values.
Definition: PatternMatch.h:560
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
Definition: PatternMatch.h:168
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
Definition: PatternMatch.h:592
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
BinOpPred_match< LHS, RHS, is_logical_shift_op > m_LogicalShift(const LHS &L, const RHS &R)
Matches logical shift operations.
match_combine_and< LTy, RTy > m_CombineAnd(const LTy &L, const RTy &R)
Combine two pattern matchers matching L && R.
Definition: PatternMatch.h:245
MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > m_SMin(const LHS &L, const RHS &R)
m_Intrinsic_Ty< Opnd0 >::Ty m_Sqrt(const Opnd0 &Op0)
BinaryOp_match< LHS, RHS, Instruction::Xor, true > m_c_Xor(const LHS &L, const RHS &R)
Matches an Xor with LHS and RHS in either order.
BinaryOp_match< LHS, RHS, Instruction::FAdd > m_FAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
Definition: PatternMatch.h:903
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
Definition: PatternMatch.h:599
apint_match m_APIntAllowPoison(const APInt *&Res)
Match APInt while allowing poison in splat vector constants.
Definition: PatternMatch.h:305
NoWrapTrunc_match< OpTy, TruncInst::NoSignedWrap > m_NSWTrunc(const OpTy &Op)
Matches trunc nsw.
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
match_combine_and< class_match< Constant >, match_unless< constantexpr_match > > m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
Definition: PatternMatch.h:864
ThreeOps_match< decltype(m_Value()), LHS, RHS, Instruction::Select, true > m_c_Select(const LHS &L, const RHS &R)
Match Select(C, LHS, RHS) or Select(C, RHS, LHS)
CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Shl, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWShl(const LHS &L, const RHS &R)
OverflowingBinaryOp_match< LHS, RHS, Instruction::Mul, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWMul(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::UDiv > m_UDiv(const LHS &L, const RHS &R)
cst_pred_ty< is_negated_power2_or_zero > m_NegatedPower2OrZero()
Match a integer or vector negated power-of-2.
Definition: PatternMatch.h:639
NoWrapTrunc_match< OpTy, TruncInst::NoUnsignedWrap > m_NUWTrunc(const OpTy &Op)
Matches trunc nuw.
cst_pred_ty< custom_checkfn< APInt > > m_CheckedInt(function_ref< bool(const APInt &)> CheckFn)
Match an integer or vector where CheckFn(ele) for each element is true.
Definition: PatternMatch.h:481
cst_pred_ty< is_lowbit_mask_or_zero > m_LowBitMaskOrZero()
Match an integer or vector with only the low bit(s) set.
Definition: PatternMatch.h:683
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
match_combine_or< BinaryOp_match< LHS, RHS, Instruction::Add >, DisjointOr_match< LHS, RHS > > m_AddLike(const LHS &L, const RHS &R)
Match either "add" or "or disjoint".
CastInst_match< OpTy, UIToFPInst > m_UIToFP(const OpTy &Op)
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
BinaryOp_match< LHS, RHS, Instruction::SDiv > m_SDiv(const LHS &L, const RHS &R)
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
Definition: PatternMatch.h:299
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
Signum_match< Val_t > m_Signum(const Val_t &V)
Matches a signum pattern.
CastInst_match< OpTy, SIToFPInst > m_SIToFP(const OpTy &Op)
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
cstfp_pred_ty< is_pos_zero_fp > m_PosZeroFP()
Match a floating-point positive zero.
Definition: PatternMatch.h:773
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
UAddWithOverflow_match< LHS_t, RHS_t, Sum_t > m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S)
Match an icmp instruction checking for unsigned overflow on addition.
m_Intrinsic_Ty< Opnd0 >::Ty m_VecReverse(const Opnd0 &Op0)
BinOpPred_match< LHS, RHS, is_irem_op > m_IRem(const LHS &L, const RHS &R)
Matches integer remainder operations.
apfloat_match m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
Definition: PatternMatch.h:316
match_combine_or< match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > >, match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > > > m_MaxOrMin(const LHS &L, const RHS &R)
CastInst_match< OpTy, FPTruncInst > m_FPTrunc(const OpTy &Op)
auto m_Undef()
Match an arbitrary undef constant.
Definition: PatternMatch.h:152
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
Definition: PatternMatch.h:612
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
ElementWiseBitCast_match< OpTy > m_ElementWiseBitCast(const OpTy &Op)
m_Intrinsic_Ty< Opnd0 >::Ty m_FAbs(const Opnd0 &Op0)
BinaryOp_match< LHS, RHS, Instruction::Mul, true > m_c_Mul(const LHS &L, const RHS &R)
Matches a Mul with LHS and RHS in either order.
CastOperator_match< OpTy, Instruction::PtrToInt > m_PtrToInt(const OpTy &Op)
Matches PtrToInt.
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
match_unless< Ty > m_Unless(const Ty &M)
Match if the inner matcher does NOT match.
Definition: PatternMatch.h:203
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
Definition: PatternMatch.h:239
cst_pred_ty< icmp_pred_with_threshold > m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold)
Match an integer or vector with every element comparing 'pred' (eg/ne/...) to Threshold.
Definition: PatternMatch.h:698
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:480
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition: STLExtras.h:854
OverflowResult
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1739
bool isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS, bool &TrueIfSigned)
Given an exploded icmp instruction, return true if the comparison only checks the sign bit.
bool isKnownNeverInfinity(const Value *V, unsigned Depth, const SimplifyQuery &SQ)
Return true if the floating-point scalar value is not an infinity or if the floating-point vector val...
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:657
bool isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL, bool OrZero=false, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given value is known to have exactly one bit set when defined.
Value * simplifyFCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q)
Given operands for an FCmpInst, fold the result or return null.
ConstantRange computeConstantRange(const Value *V, bool ForSigned, bool UseInstrInfo=true, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Determine the possible constant range of an integer or vector of integer value.
Constant * ConstantFoldExtractValueInstruction(Constant *Agg, ArrayRef< unsigned > Idxs)
Attempt to constant fold an extractvalue instruction with the specified operands and indices.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
Value * simplifyAddInst(Value *LHS, Value *RHS, bool IsNSW, bool IsNUW, const SimplifyQuery &Q)
Given operands for an Add, fold the result or return null.
Constant * ConstantFoldConstant(const Constant *C, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldConstant - Fold the constant using the specified DataLayout.
bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:340
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition: bit.h:281
Value * emitGEPOffset(IRBuilderBase *Builder, const DataLayout &DL, User *GEP, bool NoAssumptions=false)
Given a getelementptr instruction/constantexpr, emit the code necessary to compute the offset from th...
Definition: Local.cpp:22
constexpr unsigned MaxAnalysisRecursionDepth
Definition: ValueTracking.h:44
Constant * ConstantFoldUnaryOpOperand(unsigned Opcode, Constant *Op, const DataLayout &DL)
Attempt to constant fold a unary operation with the specified operand.
bool isKnownNegative(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the given value is known be negative (i.e.
SelectPatternFlavor
Specific patterns of select instructions we can match.
@ SPF_UNKNOWN
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
bool PointerMayBeCaptured(const Value *V, bool ReturnCaptures, bool StoreCaptures, unsigned MaxUsesToExplore=0)
PointerMayBeCaptured - Return true if this pointer value may be captured by the enclosing function (w...
SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
Definition: Function.cpp:1187
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1753
Value * simplifyICmpInst(CmpPredicate Pred, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for an ICmpInst, fold the result or return null.
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
Constant * ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL)
Attempt to constant fold a binary operation with the specified operands.
bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
@ Other
Any other memory.
bool isKnownNonEqual(const Value *V1, const Value *V2, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given values are known to be non-equal when defined.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
@ Add
Sum of integers.
@ UMax
Unsigned integer max implemented in terms of select(cmp()).
std::optional< DecomposedBitTest > decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate Pred, bool LookThroughTrunc=true, bool AllowNonZeroC=false)
Decompose an icmp into the form ((X & Mask) pred C) if possible.
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
DWARFExpression::Operation Op
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:217
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition: STLExtras.h:1945
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition: STLExtras.h:2087
bool isKnownNeverNaN(const Value *V, unsigned Depth, const SimplifyQuery &SQ)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
bool isKnownPositive(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the given value is known be positive (i.e.
bool isKnownNonNegative(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the give value is known to be non-negative.
std::optional< bool > isImpliedCondition(const Value *LHS, const Value *RHS, const DataLayout &DL, bool LHSIsTrue=true, unsigned Depth=0)
Return true if RHS is known to be implied true by LHS.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define NC
Definition: regutils.h:42
static constexpr roundingMode rmNearestTiesToEven
Definition: APFloat.h:297
static constexpr roundingMode rmTowardZero
Definition: APFloat.h:301
This callback is used in conjunction with PointerMayBeCaptured.
Represent subnormal handling kind for floating point instruction inputs and outputs.
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ PositiveZero
Denormals are flushed to positive zero.
static constexpr DenormalMode getIEEE()
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition: KnownBits.h:100
bool isZero() const
Returns true if value is all zero.
Definition: KnownBits.h:79
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition: KnownBits.h:234
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
Definition: KnownBits.h:266
APInt getSignedMaxValue() const
Return the maximal signed value possible given these KnownBits.
Definition: KnownBits.h:143
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition: KnownBits.h:281
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:43
bool isConstant() const
Returns true if we know the value of all bits.
Definition: KnownBits.h:53
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:240
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition: KnownBits.h:137
APInt getMinValue() const
Return the minimal unsigned value possible given these KnownBits.
Definition: KnownBits.h:121
bool isStrictlyPositive() const
Returns true if this value is known to be positive.
Definition: KnownBits.h:106
bool isNegative() const
Returns true if this value is known to be negative.
Definition: KnownBits.h:97
unsigned countMinPopulation() const
Returns the number of bits known to be one.
Definition: KnownBits.h:278
APInt getSignedMinValue() const
Return the minimal signed value possible given these KnownBits.
Definition: KnownBits.h:127
const APInt & getConstant() const
Returns the value when all bits have a known value.
Definition: KnownBits.h:59
SelectPatternFlavor Flavor
static bool isMinOrMax(SelectPatternFlavor SPF)
When implementing this min/max pattern as fcmp; select, does the fcmp have to be ordered?
const DataLayout & DL
Definition: SimplifyQuery.h:71
const Instruction * CxtI
Definition: SimplifyQuery.h:75
const DominatorTree * DT
Definition: SimplifyQuery.h:73
SimplifyQuery getWithInstruction(const Instruction *I) const
AssumptionCache * AC
Definition: SimplifyQuery.h:74
SimplifyQuery getWithoutDomCondCache() const
A MapVector that performs no allocations if smaller than a certain size.
Definition: MapVector.h:254