LLVM 20.0.0git
InstCombineCompares.cpp
Go to the documentation of this file.
1//===- InstCombineCompares.cpp --------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitICmp and visitFCmp functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/ADT/APSInt.h"
15#include "llvm/ADT/SetVector.h"
16#include "llvm/ADT/Statistic.h"
24#include "llvm/IR/DataLayout.h"
25#include "llvm/IR/InstrTypes.h"
30#include <bitset>
31
32using namespace llvm;
33using namespace PatternMatch;
34
35#define DEBUG_TYPE "instcombine"
36
37// How many times is a select replaced by one of its operands?
38STATISTIC(NumSel, "Number of select opts");
39
40
41/// Compute Result = In1+In2, returning true if the result overflowed for this
42/// type.
43static bool addWithOverflow(APInt &Result, const APInt &In1,
44 const APInt &In2, bool IsSigned = false) {
45 bool Overflow;
46 if (IsSigned)
47 Result = In1.sadd_ov(In2, Overflow);
48 else
49 Result = In1.uadd_ov(In2, Overflow);
50
51 return Overflow;
52}
53
54/// Compute Result = In1-In2, returning true if the result overflowed for this
55/// type.
56static bool subWithOverflow(APInt &Result, const APInt &In1,
57 const APInt &In2, bool IsSigned = false) {
58 bool Overflow;
59 if (IsSigned)
60 Result = In1.ssub_ov(In2, Overflow);
61 else
62 Result = In1.usub_ov(In2, Overflow);
63
64 return Overflow;
65}
66
67/// Given an icmp instruction, return true if any use of this comparison is a
68/// branch on sign bit comparison.
69static bool hasBranchUse(ICmpInst &I) {
70 for (auto *U : I.users())
71 if (isa<BranchInst>(U))
72 return true;
73 return false;
74}
75
76/// Returns true if the exploded icmp can be expressed as a signed comparison
77/// to zero and updates the predicate accordingly.
78/// The signedness of the comparison is preserved.
79/// TODO: Refactor with decomposeBitTestICmp()?
80static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) {
81 if (!ICmpInst::isSigned(Pred))
82 return false;
83
84 if (C.isZero())
85 return ICmpInst::isRelational(Pred);
86
87 if (C.isOne()) {
88 if (Pred == ICmpInst::ICMP_SLT) {
89 Pred = ICmpInst::ICMP_SLE;
90 return true;
91 }
92 } else if (C.isAllOnes()) {
93 if (Pred == ICmpInst::ICMP_SGT) {
94 Pred = ICmpInst::ICMP_SGE;
95 return true;
96 }
97 }
98
99 return false;
100}
101
102/// This is called when we see this pattern:
103/// cmp pred (load (gep GV, ...)), cmpcst
104/// where GV is a global variable with a constant initializer. Try to simplify
105/// this into some simple computation that does not need the load. For example
106/// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3".
107///
108/// If AndCst is non-null, then the loaded value is masked with that constant
109/// before doing the comparison. This handles cases like "A[i]&4 == 0".
112 ConstantInt *AndCst) {
113 if (LI->isVolatile() || LI->getType() != GEP->getResultElementType() ||
114 GV->getValueType() != GEP->getSourceElementType() || !GV->isConstant() ||
116 return nullptr;
117
119 if (!isa<ConstantArray>(Init) && !isa<ConstantDataArray>(Init))
120 return nullptr;
121
122 uint64_t ArrayElementCount = Init->getType()->getArrayNumElements();
123 // Don't blow up on huge arrays.
124 if (ArrayElementCount > MaxArraySizeForCombine)
125 return nullptr;
126
127 // There are many forms of this optimization we can handle, for now, just do
128 // the simple index into a single-dimensional array.
129 //
130 // Require: GEP GV, 0, i {{, constant indices}}
131 if (GEP->getNumOperands() < 3 || !isa<ConstantInt>(GEP->getOperand(1)) ||
132 !cast<ConstantInt>(GEP->getOperand(1))->isZero() ||
133 isa<Constant>(GEP->getOperand(2)))
134 return nullptr;
135
136 // Check that indices after the variable are constants and in-range for the
137 // type they index. Collect the indices. This is typically for arrays of
138 // structs.
139 SmallVector<unsigned, 4> LaterIndices;
140
141 Type *EltTy = Init->getType()->getArrayElementType();
142 for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) {
143 ConstantInt *Idx = dyn_cast<ConstantInt>(GEP->getOperand(i));
144 if (!Idx)
145 return nullptr; // Variable index.
146
147 uint64_t IdxVal = Idx->getZExtValue();
148 if ((unsigned)IdxVal != IdxVal)
149 return nullptr; // Too large array index.
150
151 if (StructType *STy = dyn_cast<StructType>(EltTy))
152 EltTy = STy->getElementType(IdxVal);
153 else if (ArrayType *ATy = dyn_cast<ArrayType>(EltTy)) {
154 if (IdxVal >= ATy->getNumElements())
155 return nullptr;
156 EltTy = ATy->getElementType();
157 } else {
158 return nullptr; // Unknown type.
159 }
160
161 LaterIndices.push_back(IdxVal);
162 }
163
164 enum { Overdefined = -3, Undefined = -2 };
165
166 // Variables for our state machines.
167
168 // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form
169 // "i == 47 | i == 87", where 47 is the first index the condition is true for,
170 // and 87 is the second (and last) index. FirstTrueElement is -2 when
171 // undefined, otherwise set to the first true element. SecondTrueElement is
172 // -2 when undefined, -3 when overdefined and >= 0 when that index is true.
173 int FirstTrueElement = Undefined, SecondTrueElement = Undefined;
174
175 // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the
176 // form "i != 47 & i != 87". Same state transitions as for true elements.
177 int FirstFalseElement = Undefined, SecondFalseElement = Undefined;
178
179 /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these
180 /// define a state machine that triggers for ranges of values that the index
181 /// is true or false for. This triggers on things like "abbbbc"[i] == 'b'.
182 /// This is -2 when undefined, -3 when overdefined, and otherwise the last
183 /// index in the range (inclusive). We use -2 for undefined here because we
184 /// use relative comparisons and don't want 0-1 to match -1.
185 int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined;
186
187 // MagicBitvector - This is a magic bitvector where we set a bit if the
188 // comparison is true for element 'i'. If there are 64 elements or less in
189 // the array, this will fully represent all the comparison results.
190 uint64_t MagicBitvector = 0;
191
192 // Scan the array and see if one of our patterns matches.
193 Constant *CompareRHS = cast<Constant>(ICI.getOperand(1));
194 for (unsigned i = 0, e = ArrayElementCount; i != e; ++i) {
195 Constant *Elt = Init->getAggregateElement(i);
196 if (!Elt)
197 return nullptr;
198
199 // If this is indexing an array of structures, get the structure element.
200 if (!LaterIndices.empty()) {
201 Elt = ConstantFoldExtractValueInstruction(Elt, LaterIndices);
202 if (!Elt)
203 return nullptr;
204 }
205
206 // If the element is masked, handle it.
207 if (AndCst) {
208 Elt = ConstantFoldBinaryOpOperands(Instruction::And, Elt, AndCst, DL);
209 if (!Elt)
210 return nullptr;
211 }
212
213 // Find out if the comparison would be true or false for the i'th element.
215 CompareRHS, DL, &TLI);
216 if (!C)
217 return nullptr;
218
219 // If the result is undef for this element, ignore it.
220 if (isa<UndefValue>(C)) {
221 // Extend range state machines to cover this element in case there is an
222 // undef in the middle of the range.
223 if (TrueRangeEnd == (int)i - 1)
224 TrueRangeEnd = i;
225 if (FalseRangeEnd == (int)i - 1)
226 FalseRangeEnd = i;
227 continue;
228 }
229
230 // If we can't compute the result for any of the elements, we have to give
231 // up evaluating the entire conditional.
232 if (!isa<ConstantInt>(C))
233 return nullptr;
234
235 // Otherwise, we know if the comparison is true or false for this element,
236 // update our state machines.
237 bool IsTrueForElt = !cast<ConstantInt>(C)->isZero();
238
239 // State machine for single/double/range index comparison.
240 if (IsTrueForElt) {
241 // Update the TrueElement state machine.
242 if (FirstTrueElement == Undefined)
243 FirstTrueElement = TrueRangeEnd = i; // First true element.
244 else {
245 // Update double-compare state machine.
246 if (SecondTrueElement == Undefined)
247 SecondTrueElement = i;
248 else
249 SecondTrueElement = Overdefined;
250
251 // Update range state machine.
252 if (TrueRangeEnd == (int)i - 1)
253 TrueRangeEnd = i;
254 else
255 TrueRangeEnd = Overdefined;
256 }
257 } else {
258 // Update the FalseElement state machine.
259 if (FirstFalseElement == Undefined)
260 FirstFalseElement = FalseRangeEnd = i; // First false element.
261 else {
262 // Update double-compare state machine.
263 if (SecondFalseElement == Undefined)
264 SecondFalseElement = i;
265 else
266 SecondFalseElement = Overdefined;
267
268 // Update range state machine.
269 if (FalseRangeEnd == (int)i - 1)
270 FalseRangeEnd = i;
271 else
272 FalseRangeEnd = Overdefined;
273 }
274 }
275
276 // If this element is in range, update our magic bitvector.
277 if (i < 64 && IsTrueForElt)
278 MagicBitvector |= 1ULL << i;
279
280 // If all of our states become overdefined, bail out early. Since the
281 // predicate is expensive, only check it every 8 elements. This is only
282 // really useful for really huge arrays.
283 if ((i & 8) == 0 && i >= 64 && SecondTrueElement == Overdefined &&
284 SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined &&
285 FalseRangeEnd == Overdefined)
286 return nullptr;
287 }
288
289 // Now that we've scanned the entire array, emit our new comparison(s). We
290 // order the state machines in complexity of the generated code.
291 Value *Idx = GEP->getOperand(2);
292
293 // If the index is larger than the pointer offset size of the target, truncate
294 // the index down like the GEP would do implicitly. We don't have to do this
295 // for an inbounds GEP because the index can't be out of range.
296 if (!GEP->isInBounds()) {
297 Type *PtrIdxTy = DL.getIndexType(GEP->getType());
298 unsigned OffsetSize = PtrIdxTy->getIntegerBitWidth();
299 if (Idx->getType()->getPrimitiveSizeInBits().getFixedValue() > OffsetSize)
300 Idx = Builder.CreateTrunc(Idx, PtrIdxTy);
301 }
302
303 // If inbounds keyword is not present, Idx * ElementSize can overflow.
304 // Let's assume that ElementSize is 2 and the wanted value is at offset 0.
305 // Then, there are two possible values for Idx to match offset 0:
306 // 0x00..00, 0x80..00.
307 // Emitting 'icmp eq Idx, 0' isn't correct in this case because the
308 // comparison is false if Idx was 0x80..00.
309 // We need to erase the highest countTrailingZeros(ElementSize) bits of Idx.
310 unsigned ElementSize =
311 DL.getTypeAllocSize(Init->getType()->getArrayElementType());
312 auto MaskIdx = [&](Value *Idx) {
313 if (!GEP->isInBounds() && llvm::countr_zero(ElementSize) != 0) {
314 Value *Mask = Constant::getAllOnesValue(Idx->getType());
315 Mask = Builder.CreateLShr(Mask, llvm::countr_zero(ElementSize));
316 Idx = Builder.CreateAnd(Idx, Mask);
317 }
318 return Idx;
319 };
320
321 // If the comparison is only true for one or two elements, emit direct
322 // comparisons.
323 if (SecondTrueElement != Overdefined) {
324 Idx = MaskIdx(Idx);
325 // None true -> false.
326 if (FirstTrueElement == Undefined)
327 return replaceInstUsesWith(ICI, Builder.getFalse());
328
329 Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement);
330
331 // True for one element -> 'i == 47'.
332 if (SecondTrueElement == Undefined)
333 return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx);
334
335 // True for two elements -> 'i == 47 | i == 72'.
336 Value *C1 = Builder.CreateICmpEQ(Idx, FirstTrueIdx);
337 Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement);
338 Value *C2 = Builder.CreateICmpEQ(Idx, SecondTrueIdx);
339 return BinaryOperator::CreateOr(C1, C2);
340 }
341
342 // If the comparison is only false for one or two elements, emit direct
343 // comparisons.
344 if (SecondFalseElement != Overdefined) {
345 Idx = MaskIdx(Idx);
346 // None false -> true.
347 if (FirstFalseElement == Undefined)
348 return replaceInstUsesWith(ICI, Builder.getTrue());
349
350 Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement);
351
352 // False for one element -> 'i != 47'.
353 if (SecondFalseElement == Undefined)
354 return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx);
355
356 // False for two elements -> 'i != 47 & i != 72'.
357 Value *C1 = Builder.CreateICmpNE(Idx, FirstFalseIdx);
358 Value *SecondFalseIdx =
359 ConstantInt::get(Idx->getType(), SecondFalseElement);
360 Value *C2 = Builder.CreateICmpNE(Idx, SecondFalseIdx);
361 return BinaryOperator::CreateAnd(C1, C2);
362 }
363
364 // If the comparison can be replaced with a range comparison for the elements
365 // where it is true, emit the range check.
366 if (TrueRangeEnd != Overdefined) {
367 assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare");
368 Idx = MaskIdx(Idx);
369
370 // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1).
371 if (FirstTrueElement) {
372 Value *Offs = ConstantInt::get(Idx->getType(), -FirstTrueElement);
373 Idx = Builder.CreateAdd(Idx, Offs);
374 }
375
376 Value *End =
377 ConstantInt::get(Idx->getType(), TrueRangeEnd - FirstTrueElement + 1);
378 return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End);
379 }
380
381 // False range check.
382 if (FalseRangeEnd != Overdefined) {
383 assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare");
384 Idx = MaskIdx(Idx);
385 // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse).
386 if (FirstFalseElement) {
387 Value *Offs = ConstantInt::get(Idx->getType(), -FirstFalseElement);
388 Idx = Builder.CreateAdd(Idx, Offs);
389 }
390
391 Value *End =
392 ConstantInt::get(Idx->getType(), FalseRangeEnd - FirstFalseElement);
393 return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End);
394 }
395
396 // If a magic bitvector captures the entire comparison state
397 // of this load, replace it with computation that does:
398 // ((magic_cst >> i) & 1) != 0
399 {
400 Type *Ty = nullptr;
401
402 // Look for an appropriate type:
403 // - The type of Idx if the magic fits
404 // - The smallest fitting legal type
405 if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth())
406 Ty = Idx->getType();
407 else
408 Ty = DL.getSmallestLegalIntType(Init->getContext(), ArrayElementCount);
409
410 if (Ty) {
411 Idx = MaskIdx(Idx);
412 Value *V = Builder.CreateIntCast(Idx, Ty, false);
413 V = Builder.CreateLShr(ConstantInt::get(Ty, MagicBitvector), V);
414 V = Builder.CreateAnd(ConstantInt::get(Ty, 1), V);
415 return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0));
416 }
417 }
418
419 return nullptr;
420}
421
422/// Returns true if we can rewrite Start as a GEP with pointer Base
423/// and some integer offset. The nodes that need to be re-written
424/// for this transformation will be added to Explored.
426 const DataLayout &DL,
427 SetVector<Value *> &Explored) {
428 SmallVector<Value *, 16> WorkList(1, Start);
429 Explored.insert(Base);
430
431 // The following traversal gives us an order which can be used
432 // when doing the final transformation. Since in the final
433 // transformation we create the PHI replacement instructions first,
434 // we don't have to get them in any particular order.
435 //
436 // However, for other instructions we will have to traverse the
437 // operands of an instruction first, which means that we have to
438 // do a post-order traversal.
439 while (!WorkList.empty()) {
441
442 while (!WorkList.empty()) {
443 if (Explored.size() >= 100)
444 return false;
445
446 Value *V = WorkList.back();
447
448 if (Explored.contains(V)) {
449 WorkList.pop_back();
450 continue;
451 }
452
453 if (!isa<GetElementPtrInst>(V) && !isa<PHINode>(V))
454 // We've found some value that we can't explore which is different from
455 // the base. Therefore we can't do this transformation.
456 return false;
457
458 if (auto *GEP = dyn_cast<GEPOperator>(V)) {
459 // Only allow inbounds GEPs with at most one variable offset.
460 auto IsNonConst = [](Value *V) { return !isa<ConstantInt>(V); };
461 if (!GEP->isInBounds() || count_if(GEP->indices(), IsNonConst) > 1)
462 return false;
463
464 NW = NW.intersectForOffsetAdd(GEP->getNoWrapFlags());
465 if (!Explored.contains(GEP->getOperand(0)))
466 WorkList.push_back(GEP->getOperand(0));
467 }
468
469 if (WorkList.back() == V) {
470 WorkList.pop_back();
471 // We've finished visiting this node, mark it as such.
472 Explored.insert(V);
473 }
474
475 if (auto *PN = dyn_cast<PHINode>(V)) {
476 // We cannot transform PHIs on unsplittable basic blocks.
477 if (isa<CatchSwitchInst>(PN->getParent()->getTerminator()))
478 return false;
479 Explored.insert(PN);
480 PHIs.insert(PN);
481 }
482 }
483
484 // Explore the PHI nodes further.
485 for (auto *PN : PHIs)
486 for (Value *Op : PN->incoming_values())
487 if (!Explored.contains(Op))
488 WorkList.push_back(Op);
489 }
490
491 // Make sure that we can do this. Since we can't insert GEPs in a basic
492 // block before a PHI node, we can't easily do this transformation if
493 // we have PHI node users of transformed instructions.
494 for (Value *Val : Explored) {
495 for (Value *Use : Val->uses()) {
496
497 auto *PHI = dyn_cast<PHINode>(Use);
498 auto *Inst = dyn_cast<Instruction>(Val);
499
500 if (Inst == Base || Inst == PHI || !Inst || !PHI ||
501 !Explored.contains(PHI))
502 continue;
503
504 if (PHI->getParent() == Inst->getParent())
505 return false;
506 }
507 }
508 return true;
509}
510
511// Sets the appropriate insert point on Builder where we can add
512// a replacement Instruction for V (if that is possible).
513static void setInsertionPoint(IRBuilder<> &Builder, Value *V,
514 bool Before = true) {
515 if (auto *PHI = dyn_cast<PHINode>(V)) {
516 BasicBlock *Parent = PHI->getParent();
517 Builder.SetInsertPoint(Parent, Parent->getFirstInsertionPt());
518 return;
519 }
520 if (auto *I = dyn_cast<Instruction>(V)) {
521 if (!Before)
522 I = &*std::next(I->getIterator());
523 Builder.SetInsertPoint(I);
524 return;
525 }
526 if (auto *A = dyn_cast<Argument>(V)) {
527 // Set the insertion point in the entry block.
528 BasicBlock &Entry = A->getParent()->getEntryBlock();
529 Builder.SetInsertPoint(&Entry, Entry.getFirstInsertionPt());
530 return;
531 }
532 // Otherwise, this is a constant and we don't need to set a new
533 // insertion point.
534 assert(isa<Constant>(V) && "Setting insertion point for unknown value!");
535}
536
537/// Returns a re-written value of Start as an indexed GEP using Base as a
538/// pointer.
540 const DataLayout &DL,
541 SetVector<Value *> &Explored,
542 InstCombiner &IC) {
543 // Perform all the substitutions. This is a bit tricky because we can
544 // have cycles in our use-def chains.
545 // 1. Create the PHI nodes without any incoming values.
546 // 2. Create all the other values.
547 // 3. Add the edges for the PHI nodes.
548 // 4. Emit GEPs to get the original pointers.
549 // 5. Remove the original instructions.
550 Type *IndexType = IntegerType::get(
551 Base->getContext(), DL.getIndexTypeSizeInBits(Start->getType()));
552
554 NewInsts[Base] = ConstantInt::getNullValue(IndexType);
555
556 // Create the new PHI nodes, without adding any incoming values.
557 for (Value *Val : Explored) {
558 if (Val == Base)
559 continue;
560 // Create empty phi nodes. This avoids cyclic dependencies when creating
561 // the remaining instructions.
562 if (auto *PHI = dyn_cast<PHINode>(Val))
563 NewInsts[PHI] =
564 PHINode::Create(IndexType, PHI->getNumIncomingValues(),
565 PHI->getName() + ".idx", PHI->getIterator());
566 }
567 IRBuilder<> Builder(Base->getContext());
568
569 // Create all the other instructions.
570 for (Value *Val : Explored) {
571 if (NewInsts.contains(Val))
572 continue;
573
574 if (auto *GEP = dyn_cast<GEPOperator>(Val)) {
575 setInsertionPoint(Builder, GEP);
576 Value *Op = NewInsts[GEP->getOperand(0)];
577 Value *OffsetV = emitGEPOffset(&Builder, DL, GEP);
578 if (isa<ConstantInt>(Op) && cast<ConstantInt>(Op)->isZero())
579 NewInsts[GEP] = OffsetV;
580 else
581 NewInsts[GEP] = Builder.CreateAdd(
582 Op, OffsetV, GEP->getOperand(0)->getName() + ".add",
583 /*NUW=*/NW.hasNoUnsignedWrap(),
584 /*NSW=*/NW.hasNoUnsignedSignedWrap());
585 continue;
586 }
587 if (isa<PHINode>(Val))
588 continue;
589
590 llvm_unreachable("Unexpected instruction type");
591 }
592
593 // Add the incoming values to the PHI nodes.
594 for (Value *Val : Explored) {
595 if (Val == Base)
596 continue;
597 // All the instructions have been created, we can now add edges to the
598 // phi nodes.
599 if (auto *PHI = dyn_cast<PHINode>(Val)) {
600 PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]);
601 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) {
602 Value *NewIncoming = PHI->getIncomingValue(I);
603
604 auto It = NewInsts.find(NewIncoming);
605 if (It != NewInsts.end())
606 NewIncoming = It->second;
607
608 NewPhi->addIncoming(NewIncoming, PHI->getIncomingBlock(I));
609 }
610 }
611 }
612
613 for (Value *Val : Explored) {
614 if (Val == Base)
615 continue;
616
617 setInsertionPoint(Builder, Val, false);
618 // Create GEP for external users.
619 Value *NewVal = Builder.CreateGEP(Builder.getInt8Ty(), Base, NewInsts[Val],
620 Val->getName() + ".ptr", NW);
621 IC.replaceInstUsesWith(*cast<Instruction>(Val), NewVal);
622 // Add old instruction to worklist for DCE. We don't directly remove it
623 // here because the original compare is one of the users.
624 IC.addToWorklist(cast<Instruction>(Val));
625 }
626
627 return NewInsts[Start];
628}
629
630/// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant.
631/// We can look through PHIs, GEPs and casts in order to determine a common base
632/// between GEPLHS and RHS.
635 const DataLayout &DL,
636 InstCombiner &IC) {
637 // FIXME: Support vector of pointers.
638 if (GEPLHS->getType()->isVectorTy())
639 return nullptr;
640
641 if (!GEPLHS->hasAllConstantIndices())
642 return nullptr;
643
644 APInt Offset(DL.getIndexTypeSizeInBits(GEPLHS->getType()), 0);
645 Value *PtrBase =
647 /*AllowNonInbounds*/ false);
648
649 // Bail if we looked through addrspacecast.
650 if (PtrBase->getType() != GEPLHS->getType())
651 return nullptr;
652
653 // The set of nodes that will take part in this transformation.
654 SetVector<Value *> Nodes;
655 GEPNoWrapFlags NW = GEPLHS->getNoWrapFlags();
656 if (!canRewriteGEPAsOffset(RHS, PtrBase, NW, DL, Nodes))
657 return nullptr;
658
659 // We know we can re-write this as
660 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2)
661 // Since we've only looked through inbouds GEPs we know that we
662 // can't have overflow on either side. We can therefore re-write
663 // this as:
664 // OFFSET1 cmp OFFSET2
665 Value *NewRHS = rewriteGEPAsOffset(RHS, PtrBase, NW, DL, Nodes, IC);
666
667 // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written
668 // GEP having PtrBase as the pointer base, and has returned in NewRHS the
669 // offset. Since Index is the offset of LHS to the base pointer, we will now
670 // compare the offsets instead of comparing the pointers.
672 IC.Builder.getInt(Offset), NewRHS);
673}
674
675/// Fold comparisons between a GEP instruction and something else. At this point
676/// we know that the GEP is on the LHS of the comparison.
679 // Don't transform signed compares of GEPs into index compares. Even if the
680 // GEP is inbounds, the final add of the base pointer can have signed overflow
681 // and would change the result of the icmp.
682 // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be
683 // the maximum signed value for the pointer type.
685 return nullptr;
686
687 // Look through bitcasts and addrspacecasts. We do not however want to remove
688 // 0 GEPs.
689 if (!isa<GetElementPtrInst>(RHS))
691
692 auto CanFold = [Cond](GEPNoWrapFlags NW) {
694 return true;
695
696 // Unsigned predicates can be folded if the GEPs have *any* nowrap flags.
698 return NW != GEPNoWrapFlags::none();
699 };
700
701 auto NewICmp = [Cond](GEPNoWrapFlags NW, Value *Op1, Value *Op2) {
702 if (!NW.hasNoUnsignedWrap()) {
703 // Convert signed to unsigned comparison.
704 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Op1, Op2);
705 }
706
707 auto *I = new ICmpInst(Cond, Op1, Op2);
708 I->setSameSign(NW.hasNoUnsignedSignedWrap());
709 return I;
710 };
711
712 Value *PtrBase = GEPLHS->getOperand(0);
713 if (PtrBase == RHS && CanFold(GEPLHS->getNoWrapFlags())) {
714 // ((gep Ptr, OFFSET) cmp Ptr) ---> (OFFSET cmp 0).
715 Value *Offset = EmitGEPOffset(GEPLHS);
716 return NewICmp(GEPLHS->getNoWrapFlags(), Offset,
717 Constant::getNullValue(Offset->getType()));
718 }
719
720 if (GEPLHS->isInBounds() && ICmpInst::isEquality(Cond) &&
721 isa<Constant>(RHS) && cast<Constant>(RHS)->isNullValue() &&
722 !NullPointerIsDefined(I.getFunction(),
724 // For most address spaces, an allocation can't be placed at null, but null
725 // itself is treated as a 0 size allocation in the in bounds rules. Thus,
726 // the only valid inbounds address derived from null, is null itself.
727 // Thus, we have four cases to consider:
728 // 1) Base == nullptr, Offset == 0 -> inbounds, null
729 // 2) Base == nullptr, Offset != 0 -> poison as the result is out of bounds
730 // 3) Base != nullptr, Offset == (-base) -> poison (crossing allocations)
731 // 4) Base != nullptr, Offset != (-base) -> nonnull (and possibly poison)
732 //
733 // (Note if we're indexing a type of size 0, that simply collapses into one
734 // of the buckets above.)
735 //
736 // In general, we're allowed to make values less poison (i.e. remove
737 // sources of full UB), so in this case, we just select between the two
738 // non-poison cases (1 and 4 above).
739 //
740 // For vectors, we apply the same reasoning on a per-lane basis.
741 auto *Base = GEPLHS->getPointerOperand();
742 if (GEPLHS->getType()->isVectorTy() && Base->getType()->isPointerTy()) {
743 auto EC = cast<VectorType>(GEPLHS->getType())->getElementCount();
745 }
746 return new ICmpInst(Cond, Base,
748 cast<Constant>(RHS), Base->getType()));
749 } else if (GEPOperator *GEPRHS = dyn_cast<GEPOperator>(RHS)) {
750 GEPNoWrapFlags NW = GEPLHS->getNoWrapFlags() & GEPRHS->getNoWrapFlags();
751
752 // If the base pointers are different, but the indices are the same, just
753 // compare the base pointer.
754 if (PtrBase != GEPRHS->getOperand(0)) {
755 bool IndicesTheSame =
756 GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
757 GEPLHS->getPointerOperand()->getType() ==
758 GEPRHS->getPointerOperand()->getType() &&
759 GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType();
760 if (IndicesTheSame)
761 for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i)
762 if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) {
763 IndicesTheSame = false;
764 break;
765 }
766
767 // If all indices are the same, just compare the base pointers.
768 Type *BaseType = GEPLHS->getOperand(0)->getType();
769 if (IndicesTheSame &&
770 CmpInst::makeCmpResultType(BaseType) == I.getType() && CanFold(NW))
771 return new ICmpInst(Cond, GEPLHS->getOperand(0), GEPRHS->getOperand(0));
772
773 // If we're comparing GEPs with two base pointers that only differ in type
774 // and both GEPs have only constant indices or just one use, then fold
775 // the compare with the adjusted indices.
776 // FIXME: Support vector of pointers.
777 if (GEPLHS->isInBounds() && GEPRHS->isInBounds() &&
778 (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) &&
779 (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) &&
780 PtrBase->stripPointerCasts() ==
781 GEPRHS->getOperand(0)->stripPointerCasts() &&
782 !GEPLHS->getType()->isVectorTy()) {
783 Value *LOffset = EmitGEPOffset(GEPLHS);
784 Value *ROffset = EmitGEPOffset(GEPRHS);
785
786 // If we looked through an addrspacecast between different sized address
787 // spaces, the LHS and RHS pointers are different sized
788 // integers. Truncate to the smaller one.
789 Type *LHSIndexTy = LOffset->getType();
790 Type *RHSIndexTy = ROffset->getType();
791 if (LHSIndexTy != RHSIndexTy) {
792 if (LHSIndexTy->getPrimitiveSizeInBits().getFixedValue() <
793 RHSIndexTy->getPrimitiveSizeInBits().getFixedValue()) {
794 ROffset = Builder.CreateTrunc(ROffset, LHSIndexTy);
795 } else
796 LOffset = Builder.CreateTrunc(LOffset, RHSIndexTy);
797 }
798
800 LOffset, ROffset);
801 return replaceInstUsesWith(I, Cmp);
802 }
803
804 // Otherwise, the base pointers are different and the indices are
805 // different. Try convert this to an indexed compare by looking through
806 // PHIs/casts.
807 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL, *this);
808 }
809
810 if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
811 GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType()) {
812 // If the GEPs only differ by one index, compare it.
813 unsigned NumDifferences = 0; // Keep track of # differences.
814 unsigned DiffOperand = 0; // The operand that differs.
815 for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i)
816 if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) {
817 Type *LHSType = GEPLHS->getOperand(i)->getType();
818 Type *RHSType = GEPRHS->getOperand(i)->getType();
819 // FIXME: Better support for vector of pointers.
820 if (LHSType->getPrimitiveSizeInBits() !=
821 RHSType->getPrimitiveSizeInBits() ||
822 (GEPLHS->getType()->isVectorTy() &&
823 (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) {
824 // Irreconcilable differences.
825 NumDifferences = 2;
826 break;
827 }
828
829 if (NumDifferences++) break;
830 DiffOperand = i;
831 }
832
833 if (NumDifferences == 0) // SAME GEP?
834 return replaceInstUsesWith(I, // No comparison is needed here.
835 ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond)));
836
837 else if (NumDifferences == 1 && CanFold(NW)) {
838 Value *LHSV = GEPLHS->getOperand(DiffOperand);
839 Value *RHSV = GEPRHS->getOperand(DiffOperand);
840 return NewICmp(NW, LHSV, RHSV);
841 }
842 }
843
844 if (CanFold(NW)) {
845 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2)
846 Value *L = EmitGEPOffset(GEPLHS, /*RewriteGEP=*/true);
847 Value *R = EmitGEPOffset(GEPRHS, /*RewriteGEP=*/true);
848 return NewICmp(NW, L, R);
849 }
850 }
851
852 // Try convert this to an indexed compare by looking through PHIs/casts as a
853 // last resort.
854 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL, *this);
855}
856
858 // It would be tempting to fold away comparisons between allocas and any
859 // pointer not based on that alloca (e.g. an argument). However, even
860 // though such pointers cannot alias, they can still compare equal.
861 //
862 // But LLVM doesn't specify where allocas get their memory, so if the alloca
863 // doesn't escape we can argue that it's impossible to guess its value, and we
864 // can therefore act as if any such guesses are wrong.
865 //
866 // However, we need to ensure that this folding is consistent: We can't fold
867 // one comparison to false, and then leave a different comparison against the
868 // same value alone (as it might evaluate to true at runtime, leading to a
869 // contradiction). As such, this code ensures that all comparisons are folded
870 // at the same time, and there are no other escapes.
871
872 struct CmpCaptureTracker : public CaptureTracker {
873 AllocaInst *Alloca;
874 bool Captured = false;
875 /// The value of the map is a bit mask of which icmp operands the alloca is
876 /// used in.
878
879 CmpCaptureTracker(AllocaInst *Alloca) : Alloca(Alloca) {}
880
881 void tooManyUses() override { Captured = true; }
882
883 bool captured(const Use *U) override {
884 auto *ICmp = dyn_cast<ICmpInst>(U->getUser());
885 // We need to check that U is based *only* on the alloca, and doesn't
886 // have other contributions from a select/phi operand.
887 // TODO: We could check whether getUnderlyingObjects() reduces to one
888 // object, which would allow looking through phi nodes.
889 if (ICmp && ICmp->isEquality() && getUnderlyingObject(*U) == Alloca) {
890 // Collect equality icmps of the alloca, and don't treat them as
891 // captures.
892 ICmps[ICmp] |= 1u << U->getOperandNo();
893 return false;
894 }
895
896 Captured = true;
897 return true;
898 }
899 };
900
901 CmpCaptureTracker Tracker(Alloca);
902 PointerMayBeCaptured(Alloca, &Tracker);
903 if (Tracker.Captured)
904 return false;
905
906 bool Changed = false;
907 for (auto [ICmp, Operands] : Tracker.ICmps) {
908 switch (Operands) {
909 case 1:
910 case 2: {
911 // The alloca is only used in one icmp operand. Assume that the
912 // equality is false.
913 auto *Res = ConstantInt::get(
914 ICmp->getType(), ICmp->getPredicate() == ICmpInst::ICMP_NE);
915 replaceInstUsesWith(*ICmp, Res);
917 Changed = true;
918 break;
919 }
920 case 3:
921 // Both icmp operands are based on the alloca, so this is comparing
922 // pointer offsets, without leaking any information about the address
923 // of the alloca. Ignore such comparisons.
924 break;
925 default:
926 llvm_unreachable("Cannot happen");
927 }
928 }
929
930 return Changed;
931}
932
933/// Fold "icmp pred (X+C), X".
935 CmpPredicate Pred) {
936 // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0,
937 // so the values can never be equal. Similarly for all other "or equals"
938 // operators.
939 assert(!!C && "C should not be zero!");
940
941 // (X+1) <u X --> X >u (MAXUINT-1) --> X == 255
942 // (X+2) <u X --> X >u (MAXUINT-2) --> X > 253
943 // (X+MAXUINT) <u X --> X >u (MAXUINT-MAXUINT) --> X != 0
944 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
945 Constant *R = ConstantInt::get(X->getType(),
946 APInt::getMaxValue(C.getBitWidth()) - C);
947 return new ICmpInst(ICmpInst::ICMP_UGT, X, R);
948 }
949
950 // (X+1) >u X --> X <u (0-1) --> X != 255
951 // (X+2) >u X --> X <u (0-2) --> X <u 254
952 // (X+MAXUINT) >u X --> X <u (0-MAXUINT) --> X <u 1 --> X == 0
953 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
954 return new ICmpInst(ICmpInst::ICMP_ULT, X,
955 ConstantInt::get(X->getType(), -C));
956
957 APInt SMax = APInt::getSignedMaxValue(C.getBitWidth());
958
959 // (X+ 1) <s X --> X >s (MAXSINT-1) --> X == 127
960 // (X+ 2) <s X --> X >s (MAXSINT-2) --> X >s 125
961 // (X+MAXSINT) <s X --> X >s (MAXSINT-MAXSINT) --> X >s 0
962 // (X+MINSINT) <s X --> X >s (MAXSINT-MINSINT) --> X >s -1
963 // (X+ -2) <s X --> X >s (MAXSINT- -2) --> X >s 126
964 // (X+ -1) <s X --> X >s (MAXSINT- -1) --> X != 127
965 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
966 return new ICmpInst(ICmpInst::ICMP_SGT, X,
967 ConstantInt::get(X->getType(), SMax - C));
968
969 // (X+ 1) >s X --> X <s (MAXSINT-(1-1)) --> X != 127
970 // (X+ 2) >s X --> X <s (MAXSINT-(2-1)) --> X <s 126
971 // (X+MAXSINT) >s X --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1
972 // (X+MINSINT) >s X --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2
973 // (X+ -2) >s X --> X <s (MAXSINT-(-2-1)) --> X <s -126
974 // (X+ -1) >s X --> X <s (MAXSINT-(-1-1)) --> X == -128
975
976 assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE);
977 return new ICmpInst(ICmpInst::ICMP_SLT, X,
978 ConstantInt::get(X->getType(), SMax - (C - 1)));
979}
980
981/// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" ->
982/// (icmp eq/ne A, Log2(AP2/AP1)) ->
983/// (icmp eq/ne A, Log2(AP2) - Log2(AP1)).
985 const APInt &AP1,
986 const APInt &AP2) {
987 assert(I.isEquality() && "Cannot fold icmp gt/lt");
988
989 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
990 if (I.getPredicate() == I.ICMP_NE)
991 Pred = CmpInst::getInversePredicate(Pred);
992 return new ICmpInst(Pred, LHS, RHS);
993 };
994
995 // Don't bother doing any work for cases which InstSimplify handles.
996 if (AP2.isZero())
997 return nullptr;
998
999 bool IsAShr = isa<AShrOperator>(I.getOperand(0));
1000 if (IsAShr) {
1001 if (AP2.isAllOnes())
1002 return nullptr;
1003 if (AP2.isNegative() != AP1.isNegative())
1004 return nullptr;
1005 if (AP2.sgt(AP1))
1006 return nullptr;
1007 }
1008
1009 if (!AP1)
1010 // 'A' must be large enough to shift out the highest set bit.
1011 return getICmp(I.ICMP_UGT, A,
1012 ConstantInt::get(A->getType(), AP2.logBase2()));
1013
1014 if (AP1 == AP2)
1015 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType()));
1016
1017 int Shift;
1018 if (IsAShr && AP1.isNegative())
1019 Shift = AP1.countl_one() - AP2.countl_one();
1020 else
1021 Shift = AP1.countl_zero() - AP2.countl_zero();
1022
1023 if (Shift > 0) {
1024 if (IsAShr && AP1 == AP2.ashr(Shift)) {
1025 // There are multiple solutions if we are comparing against -1 and the LHS
1026 // of the ashr is not a power of two.
1027 if (AP1.isAllOnes() && !AP2.isPowerOf2())
1028 return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift));
1029 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1030 } else if (AP1 == AP2.lshr(Shift)) {
1031 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1032 }
1033 }
1034
1035 // Shifting const2 will never be equal to const1.
1036 // FIXME: This should always be handled by InstSimplify?
1037 auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE);
1038 return replaceInstUsesWith(I, TorF);
1039}
1040
1041/// Handle "(icmp eq/ne (shl AP2, A), AP1)" ->
1042/// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)).
1044 const APInt &AP1,
1045 const APInt &AP2) {
1046 assert(I.isEquality() && "Cannot fold icmp gt/lt");
1047
1048 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
1049 if (I.getPredicate() == I.ICMP_NE)
1050 Pred = CmpInst::getInversePredicate(Pred);
1051 return new ICmpInst(Pred, LHS, RHS);
1052 };
1053
1054 // Don't bother doing any work for cases which InstSimplify handles.
1055 if (AP2.isZero())
1056 return nullptr;
1057
1058 unsigned AP2TrailingZeros = AP2.countr_zero();
1059
1060 if (!AP1 && AP2TrailingZeros != 0)
1061 return getICmp(
1062 I.ICMP_UGE, A,
1063 ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros));
1064
1065 if (AP1 == AP2)
1066 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType()));
1067
1068 // Get the distance between the lowest bits that are set.
1069 int Shift = AP1.countr_zero() - AP2TrailingZeros;
1070
1071 if (Shift > 0 && AP2.shl(Shift) == AP1)
1072 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1073
1074 // Shifting const2 will never be equal to const1.
1075 // FIXME: This should always be handled by InstSimplify?
1076 auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE);
1077 return replaceInstUsesWith(I, TorF);
1078}
1079
1080/// The caller has matched a pattern of the form:
1081/// I = icmp ugt (add (add A, B), CI2), CI1
1082/// If this is of the form:
1083/// sum = a + b
1084/// if (sum+128 >u 255)
1085/// Then replace it with llvm.sadd.with.overflow.i8.
1086///
1088 ConstantInt *CI2, ConstantInt *CI1,
1089 InstCombinerImpl &IC) {
1090 // The transformation we're trying to do here is to transform this into an
1091 // llvm.sadd.with.overflow. To do this, we have to replace the original add
1092 // with a narrower add, and discard the add-with-constant that is part of the
1093 // range check (if we can't eliminate it, this isn't profitable).
1094
1095 // In order to eliminate the add-with-constant, the compare can be its only
1096 // use.
1097 Instruction *AddWithCst = cast<Instruction>(I.getOperand(0));
1098 if (!AddWithCst->hasOneUse())
1099 return nullptr;
1100
1101 // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow.
1102 if (!CI2->getValue().isPowerOf2())
1103 return nullptr;
1104 unsigned NewWidth = CI2->getValue().countr_zero();
1105 if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31)
1106 return nullptr;
1107
1108 // The width of the new add formed is 1 more than the bias.
1109 ++NewWidth;
1110
1111 // Check to see that CI1 is an all-ones value with NewWidth bits.
1112 if (CI1->getBitWidth() == NewWidth ||
1113 CI1->getValue() != APInt::getLowBitsSet(CI1->getBitWidth(), NewWidth))
1114 return nullptr;
1115
1116 // This is only really a signed overflow check if the inputs have been
1117 // sign-extended; check for that condition. For example, if CI2 is 2^31 and
1118 // the operands of the add are 64 bits wide, we need at least 33 sign bits.
1119 if (IC.ComputeMaxSignificantBits(A, 0, &I) > NewWidth ||
1120 IC.ComputeMaxSignificantBits(B, 0, &I) > NewWidth)
1121 return nullptr;
1122
1123 // In order to replace the original add with a narrower
1124 // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant
1125 // and truncates that discard the high bits of the add. Verify that this is
1126 // the case.
1127 Instruction *OrigAdd = cast<Instruction>(AddWithCst->getOperand(0));
1128 for (User *U : OrigAdd->users()) {
1129 if (U == AddWithCst)
1130 continue;
1131
1132 // Only accept truncates for now. We would really like a nice recursive
1133 // predicate like SimplifyDemandedBits, but which goes downwards the use-def
1134 // chain to see which bits of a value are actually demanded. If the
1135 // original add had another add which was then immediately truncated, we
1136 // could still do the transformation.
1137 TruncInst *TI = dyn_cast<TruncInst>(U);
1138 if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth)
1139 return nullptr;
1140 }
1141
1142 // If the pattern matches, truncate the inputs to the narrower type and
1143 // use the sadd_with_overflow intrinsic to efficiently compute both the
1144 // result and the overflow bit.
1145 Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth);
1147 I.getModule(), Intrinsic::sadd_with_overflow, NewType);
1148
1149 InstCombiner::BuilderTy &Builder = IC.Builder;
1150
1151 // Put the new code above the original add, in case there are any uses of the
1152 // add between the add and the compare.
1153 Builder.SetInsertPoint(OrigAdd);
1154
1155 Value *TruncA = Builder.CreateTrunc(A, NewType, A->getName() + ".trunc");
1156 Value *TruncB = Builder.CreateTrunc(B, NewType, B->getName() + ".trunc");
1157 CallInst *Call = Builder.CreateCall(F, {TruncA, TruncB}, "sadd");
1158 Value *Add = Builder.CreateExtractValue(Call, 0, "sadd.result");
1159 Value *ZExt = Builder.CreateZExt(Add, OrigAdd->getType());
1160
1161 // The inner add was the result of the narrow add, zero extended to the
1162 // wider type. Replace it with the result computed by the intrinsic.
1163 IC.replaceInstUsesWith(*OrigAdd, ZExt);
1164 IC.eraseInstFromFunction(*OrigAdd);
1165
1166 // The original icmp gets replaced with the overflow value.
1167 return ExtractValueInst::Create(Call, 1, "sadd.overflow");
1168}
1169
1170/// If we have:
1171/// icmp eq/ne (urem/srem %x, %y), 0
1172/// iff %y is a power-of-two, we can replace this with a bit test:
1173/// icmp eq/ne (and %x, (add %y, -1)), 0
1175 // This fold is only valid for equality predicates.
1176 if (!I.isEquality())
1177 return nullptr;
1178 CmpPredicate Pred;
1179 Value *X, *Y, *Zero;
1180 if (!match(&I, m_ICmp(Pred, m_OneUse(m_IRem(m_Value(X), m_Value(Y))),
1181 m_CombineAnd(m_Zero(), m_Value(Zero)))))
1182 return nullptr;
1183 if (!isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, 0, &I))
1184 return nullptr;
1185 // This may increase instruction count, we don't enforce that Y is a constant.
1186 Value *Mask = Builder.CreateAdd(Y, Constant::getAllOnesValue(Y->getType()));
1187 Value *Masked = Builder.CreateAnd(X, Mask);
1188 return ICmpInst::Create(Instruction::ICmp, Pred, Masked, Zero);
1189}
1190
1191/// Fold equality-comparison between zero and any (maybe truncated) right-shift
1192/// by one-less-than-bitwidth into a sign test on the original value.
1194 Instruction *Val;
1195 CmpPredicate Pred;
1196 if (!I.isEquality() || !match(&I, m_ICmp(Pred, m_Instruction(Val), m_Zero())))
1197 return nullptr;
1198
1199 Value *X;
1200 Type *XTy;
1201
1202 Constant *C;
1203 if (match(Val, m_TruncOrSelf(m_Shr(m_Value(X), m_Constant(C))))) {
1204 XTy = X->getType();
1205 unsigned XBitWidth = XTy->getScalarSizeInBits();
1207 APInt(XBitWidth, XBitWidth - 1))))
1208 return nullptr;
1209 } else if (isa<BinaryOperator>(Val) &&
1211 cast<BinaryOperator>(Val), SQ.getWithInstruction(Val),
1212 /*AnalyzeForSignBitExtraction=*/true))) {
1213 XTy = X->getType();
1214 } else
1215 return nullptr;
1216
1217 return ICmpInst::Create(Instruction::ICmp,
1221}
1222
1223// Handle icmp pred X, 0
1225 CmpInst::Predicate Pred = Cmp.getPredicate();
1226 if (!match(Cmp.getOperand(1), m_Zero()))
1227 return nullptr;
1228
1229 // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0)
1230 if (Pred == ICmpInst::ICMP_SGT) {
1231 Value *A, *B;
1232 if (match(Cmp.getOperand(0), m_SMin(m_Value(A), m_Value(B)))) {
1234 return new ICmpInst(Pred, B, Cmp.getOperand(1));
1236 return new ICmpInst(Pred, A, Cmp.getOperand(1));
1237 }
1238 }
1239
1241 return New;
1242
1243 // Given:
1244 // icmp eq/ne (urem %x, %y), 0
1245 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
1246 // icmp eq/ne %x, 0
1247 Value *X, *Y;
1248 if (match(Cmp.getOperand(0), m_URem(m_Value(X), m_Value(Y))) &&
1249 ICmpInst::isEquality(Pred)) {
1250 KnownBits XKnown = computeKnownBits(X, 0, &Cmp);
1251 KnownBits YKnown = computeKnownBits(Y, 0, &Cmp);
1252 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
1253 return new ICmpInst(Pred, X, Cmp.getOperand(1));
1254 }
1255
1256 // (icmp eq/ne (mul X Y)) -> (icmp eq/ne X/Y) if we know about whether X/Y are
1257 // odd/non-zero/there is no overflow.
1258 if (match(Cmp.getOperand(0), m_Mul(m_Value(X), m_Value(Y))) &&
1259 ICmpInst::isEquality(Pred)) {
1260
1261 KnownBits XKnown = computeKnownBits(X, 0, &Cmp);
1262 // if X % 2 != 0
1263 // (icmp eq/ne Y)
1264 if (XKnown.countMaxTrailingZeros() == 0)
1265 return new ICmpInst(Pred, Y, Cmp.getOperand(1));
1266
1267 KnownBits YKnown = computeKnownBits(Y, 0, &Cmp);
1268 // if Y % 2 != 0
1269 // (icmp eq/ne X)
1270 if (YKnown.countMaxTrailingZeros() == 0)
1271 return new ICmpInst(Pred, X, Cmp.getOperand(1));
1272
1273 auto *BO0 = cast<OverflowingBinaryOperator>(Cmp.getOperand(0));
1274 if (BO0->hasNoUnsignedWrap() || BO0->hasNoSignedWrap()) {
1275 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
1276 // `isKnownNonZero` does more analysis than just `!KnownBits.One.isZero()`
1277 // but to avoid unnecessary work, first just if this is an obvious case.
1278
1279 // if X non-zero and NoOverflow(X * Y)
1280 // (icmp eq/ne Y)
1281 if (!XKnown.One.isZero() || isKnownNonZero(X, Q))
1282 return new ICmpInst(Pred, Y, Cmp.getOperand(1));
1283
1284 // if Y non-zero and NoOverflow(X * Y)
1285 // (icmp eq/ne X)
1286 if (!YKnown.One.isZero() || isKnownNonZero(Y, Q))
1287 return new ICmpInst(Pred, X, Cmp.getOperand(1));
1288 }
1289 // Note, we are skipping cases:
1290 // if Y % 2 != 0 AND X % 2 != 0
1291 // (false/true)
1292 // if X non-zero and Y non-zero and NoOverflow(X * Y)
1293 // (false/true)
1294 // Those can be simplified later as we would have already replaced the (icmp
1295 // eq/ne (mul X, Y)) with (icmp eq/ne X/Y) and if X/Y is known non-zero that
1296 // will fold to a constant elsewhere.
1297 }
1298 return nullptr;
1299}
1300
1301/// Fold icmp Pred X, C.
1302/// TODO: This code structure does not make sense. The saturating add fold
1303/// should be moved to some other helper and extended as noted below (it is also
1304/// possible that code has been made unnecessary - do we canonicalize IR to
1305/// overflow/saturating intrinsics or not?).
1307 // Match the following pattern, which is a common idiom when writing
1308 // overflow-safe integer arithmetic functions. The source performs an addition
1309 // in wider type and explicitly checks for overflow using comparisons against
1310 // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic.
1311 //
1312 // TODO: This could probably be generalized to handle other overflow-safe
1313 // operations if we worked out the formulas to compute the appropriate magic
1314 // constants.
1315 //
1316 // sum = a + b
1317 // if (sum+128 >u 255) ... -> llvm.sadd.with.overflow.i8
1318 CmpInst::Predicate Pred = Cmp.getPredicate();
1319 Value *Op0 = Cmp.getOperand(0), *Op1 = Cmp.getOperand(1);
1320 Value *A, *B;
1321 ConstantInt *CI, *CI2; // I = icmp ugt (add (add A, B), CI2), CI
1322 if (Pred == ICmpInst::ICMP_UGT && match(Op1, m_ConstantInt(CI)) &&
1323 match(Op0, m_Add(m_Add(m_Value(A), m_Value(B)), m_ConstantInt(CI2))))
1324 if (Instruction *Res = processUGT_ADDCST_ADD(Cmp, A, B, CI2, CI, *this))
1325 return Res;
1326
1327 // icmp(phi(C1, C2, ...), C) -> phi(icmp(C1, C), icmp(C2, C), ...).
1328 Constant *C = dyn_cast<Constant>(Op1);
1329 if (!C)
1330 return nullptr;
1331
1332 if (auto *Phi = dyn_cast<PHINode>(Op0))
1333 if (all_of(Phi->operands(), [](Value *V) { return isa<Constant>(V); })) {
1335 for (Value *V : Phi->incoming_values()) {
1336 Constant *Res =
1337 ConstantFoldCompareInstOperands(Pred, cast<Constant>(V), C, DL);
1338 if (!Res)
1339 return nullptr;
1340 Ops.push_back(Res);
1341 }
1343 PHINode *NewPhi = Builder.CreatePHI(Cmp.getType(), Phi->getNumOperands());
1344 for (auto [V, Pred] : zip(Ops, Phi->blocks()))
1345 NewPhi->addIncoming(V, Pred);
1346 return replaceInstUsesWith(Cmp, NewPhi);
1347 }
1348
1350 return R;
1351
1352 return nullptr;
1353}
1354
1355/// Canonicalize icmp instructions based on dominating conditions.
1357 // We already checked simple implication in InstSimplify, only handle complex
1358 // cases here.
1359 Value *X = Cmp.getOperand(0), *Y = Cmp.getOperand(1);
1360 const APInt *C;
1361 if (!match(Y, m_APInt(C)))
1362 return nullptr;
1363
1364 CmpInst::Predicate Pred = Cmp.getPredicate();
1366
1367 auto handleDomCond = [&](ICmpInst::Predicate DomPred,
1368 const APInt *DomC) -> Instruction * {
1369 // We have 2 compares of a variable with constants. Calculate the constant
1370 // ranges of those compares to see if we can transform the 2nd compare:
1371 // DomBB:
1372 // DomCond = icmp DomPred X, DomC
1373 // br DomCond, CmpBB, FalseBB
1374 // CmpBB:
1375 // Cmp = icmp Pred X, C
1376 ConstantRange DominatingCR =
1377 ConstantRange::makeExactICmpRegion(DomPred, *DomC);
1378 ConstantRange Intersection = DominatingCR.intersectWith(CR);
1379 ConstantRange Difference = DominatingCR.difference(CR);
1380 if (Intersection.isEmptySet())
1381 return replaceInstUsesWith(Cmp, Builder.getFalse());
1382 if (Difference.isEmptySet())
1383 return replaceInstUsesWith(Cmp, Builder.getTrue());
1384
1385 // Canonicalizing a sign bit comparison that gets used in a branch,
1386 // pessimizes codegen by generating branch on zero instruction instead
1387 // of a test and branch. So we avoid canonicalizing in such situations
1388 // because test and branch instruction has better branch displacement
1389 // than compare and branch instruction.
1390 bool UnusedBit;
1391 bool IsSignBit = isSignBitCheck(Pred, *C, UnusedBit);
1392 if (Cmp.isEquality() || (IsSignBit && hasBranchUse(Cmp)))
1393 return nullptr;
1394
1395 // Avoid an infinite loop with min/max canonicalization.
1396 // TODO: This will be unnecessary if we canonicalize to min/max intrinsics.
1397 if (Cmp.hasOneUse() &&
1398 match(Cmp.user_back(), m_MaxOrMin(m_Value(), m_Value())))
1399 return nullptr;
1400
1401 if (const APInt *EqC = Intersection.getSingleElement())
1402 return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(*EqC));
1403 if (const APInt *NeC = Difference.getSingleElement())
1404 return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(*NeC));
1405 return nullptr;
1406 };
1407
1408 for (BranchInst *BI : DC.conditionsFor(X)) {
1409 CmpPredicate DomPred;
1410 const APInt *DomC;
1411 if (!match(BI->getCondition(),
1412 m_ICmp(DomPred, m_Specific(X), m_APInt(DomC))))
1413 continue;
1414
1415 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0));
1416 if (DT.dominates(Edge0, Cmp.getParent())) {
1417 if (auto *V = handleDomCond(DomPred, DomC))
1418 return V;
1419 } else {
1420 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1));
1421 if (DT.dominates(Edge1, Cmp.getParent()))
1422 if (auto *V =
1423 handleDomCond(CmpInst::getInversePredicate(DomPred), DomC))
1424 return V;
1425 }
1426 }
1427
1428 return nullptr;
1429}
1430
1431/// Fold icmp (trunc X), C.
1433 TruncInst *Trunc,
1434 const APInt &C) {
1435 ICmpInst::Predicate Pred = Cmp.getPredicate();
1436 Value *X = Trunc->getOperand(0);
1437 Type *SrcTy = X->getType();
1438 unsigned DstBits = Trunc->getType()->getScalarSizeInBits(),
1439 SrcBits = SrcTy->getScalarSizeInBits();
1440
1441 // Match (icmp pred (trunc nuw/nsw X), C)
1442 // Which we can convert to (icmp pred X, (sext/zext C))
1443 if (shouldChangeType(Trunc->getType(), SrcTy)) {
1444 if (Trunc->hasNoSignedWrap())
1445 return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, C.sext(SrcBits)));
1446 if (!Cmp.isSigned() && Trunc->hasNoUnsignedWrap())
1447 return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, C.zext(SrcBits)));
1448 }
1449
1450 if (C.isOne() && C.getBitWidth() > 1) {
1451 // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1
1452 Value *V = nullptr;
1453 if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V))))
1454 return new ICmpInst(ICmpInst::ICMP_SLT, V,
1455 ConstantInt::get(V->getType(), 1));
1456 }
1457
1458 // TODO: Handle any shifted constant by subtracting trailing zeros.
1459 // TODO: Handle non-equality predicates.
1460 Value *Y;
1461 if (Cmp.isEquality() && match(X, m_Shl(m_One(), m_Value(Y)))) {
1462 // (trunc (1 << Y) to iN) == 0 --> Y u>= N
1463 // (trunc (1 << Y) to iN) != 0 --> Y u< N
1464 if (C.isZero()) {
1465 auto NewPred = (Pred == Cmp.ICMP_EQ) ? Cmp.ICMP_UGE : Cmp.ICMP_ULT;
1466 return new ICmpInst(NewPred, Y, ConstantInt::get(SrcTy, DstBits));
1467 }
1468 // (trunc (1 << Y) to iN) == 2**C --> Y == C
1469 // (trunc (1 << Y) to iN) != 2**C --> Y != C
1470 if (C.isPowerOf2())
1471 return new ICmpInst(Pred, Y, ConstantInt::get(SrcTy, C.logBase2()));
1472 }
1473
1474 if (Cmp.isEquality() && Trunc->hasOneUse()) {
1475 // Canonicalize to a mask and wider compare if the wide type is suitable:
1476 // (trunc X to i8) == C --> (X & 0xff) == (zext C)
1477 if (!SrcTy->isVectorTy() && shouldChangeType(DstBits, SrcBits)) {
1478 Constant *Mask =
1479 ConstantInt::get(SrcTy, APInt::getLowBitsSet(SrcBits, DstBits));
1480 Value *And = Builder.CreateAnd(X, Mask);
1481 Constant *WideC = ConstantInt::get(SrcTy, C.zext(SrcBits));
1482 return new ICmpInst(Pred, And, WideC);
1483 }
1484
1485 // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all
1486 // of the high bits truncated out of x are known.
1487 KnownBits Known = computeKnownBits(X, 0, &Cmp);
1488
1489 // If all the high bits are known, we can do this xform.
1490 if ((Known.Zero | Known.One).countl_one() >= SrcBits - DstBits) {
1491 // Pull in the high bits from known-ones set.
1492 APInt NewRHS = C.zext(SrcBits);
1493 NewRHS |= Known.One & APInt::getHighBitsSet(SrcBits, SrcBits - DstBits);
1494 return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, NewRHS));
1495 }
1496 }
1497
1498 // Look through truncated right-shift of the sign-bit for a sign-bit check:
1499 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] < 0 --> ShOp < 0
1500 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] > -1 --> ShOp > -1
1501 Value *ShOp;
1502 const APInt *ShAmtC;
1503 bool TrueIfSigned;
1504 if (isSignBitCheck(Pred, C, TrueIfSigned) &&
1505 match(X, m_Shr(m_Value(ShOp), m_APInt(ShAmtC))) &&
1506 DstBits == SrcBits - ShAmtC->getZExtValue()) {
1507 return TrueIfSigned ? new ICmpInst(ICmpInst::ICMP_SLT, ShOp,
1509 : new ICmpInst(ICmpInst::ICMP_SGT, ShOp,
1511 }
1512
1513 return nullptr;
1514}
1515
1516/// Fold icmp (trunc nuw/nsw X), (trunc nuw/nsw Y).
1517/// Fold icmp (trunc nuw/nsw X), (zext/sext Y).
1520 const SimplifyQuery &Q) {
1521 Value *X, *Y;
1522 CmpPredicate Pred;
1523 bool YIsSExt = false;
1524 // Try to match icmp (trunc X), (trunc Y)
1525 if (match(&Cmp, m_ICmp(Pred, m_Trunc(m_Value(X)), m_Trunc(m_Value(Y))))) {
1526 unsigned NoWrapFlags = cast<TruncInst>(Cmp.getOperand(0))->getNoWrapKind() &
1527 cast<TruncInst>(Cmp.getOperand(1))->getNoWrapKind();
1528 if (Cmp.isSigned()) {
1529 // For signed comparisons, both truncs must be nsw.
1530 if (!(NoWrapFlags & TruncInst::NoSignedWrap))
1531 return nullptr;
1532 } else {
1533 // For unsigned and equality comparisons, either both must be nuw or
1534 // both must be nsw, we don't care which.
1535 if (!NoWrapFlags)
1536 return nullptr;
1537 }
1538
1539 if (X->getType() != Y->getType() &&
1540 (!Cmp.getOperand(0)->hasOneUse() || !Cmp.getOperand(1)->hasOneUse()))
1541 return nullptr;
1542 if (!isDesirableIntType(X->getType()->getScalarSizeInBits()) &&
1543 isDesirableIntType(Y->getType()->getScalarSizeInBits())) {
1544 std::swap(X, Y);
1545 Pred = Cmp.getSwappedPredicate(Pred);
1546 }
1547 YIsSExt = !(NoWrapFlags & TruncInst::NoUnsignedWrap);
1548 }
1549 // Try to match icmp (trunc nuw X), (zext Y)
1550 else if (!Cmp.isSigned() &&
1551 match(&Cmp, m_c_ICmp(Pred, m_NUWTrunc(m_Value(X)),
1552 m_OneUse(m_ZExt(m_Value(Y)))))) {
1553 // Can fold trunc nuw + zext for unsigned and equality predicates.
1554 }
1555 // Try to match icmp (trunc nsw X), (sext Y)
1556 else if (match(&Cmp, m_c_ICmp(Pred, m_NSWTrunc(m_Value(X)),
1558 // Can fold trunc nsw + zext/sext for all predicates.
1559 YIsSExt =
1560 isa<SExtInst>(Cmp.getOperand(0)) || isa<SExtInst>(Cmp.getOperand(1));
1561 } else
1562 return nullptr;
1563
1564 Type *TruncTy = Cmp.getOperand(0)->getType();
1565 unsigned TruncBits = TruncTy->getScalarSizeInBits();
1566
1567 // If this transform will end up changing from desirable types -> undesirable
1568 // types skip it.
1569 if (isDesirableIntType(TruncBits) &&
1570 !isDesirableIntType(X->getType()->getScalarSizeInBits()))
1571 return nullptr;
1572
1573 Value *NewY = Builder.CreateIntCast(Y, X->getType(), YIsSExt);
1574 return new ICmpInst(Pred, X, NewY);
1575}
1576
1577/// Fold icmp (xor X, Y), C.
1580 const APInt &C) {
1581 if (Instruction *I = foldICmpXorShiftConst(Cmp, Xor, C))
1582 return I;
1583
1584 Value *X = Xor->getOperand(0);
1585 Value *Y = Xor->getOperand(1);
1586 const APInt *XorC;
1587 if (!match(Y, m_APInt(XorC)))
1588 return nullptr;
1589
1590 // If this is a comparison that tests the signbit (X < 0) or (x > -1),
1591 // fold the xor.
1592 ICmpInst::Predicate Pred = Cmp.getPredicate();
1593 bool TrueIfSigned = false;
1594 if (isSignBitCheck(Cmp.getPredicate(), C, TrueIfSigned)) {
1595
1596 // If the sign bit of the XorCst is not set, there is no change to
1597 // the operation, just stop using the Xor.
1598 if (!XorC->isNegative())
1599 return replaceOperand(Cmp, 0, X);
1600
1601 // Emit the opposite comparison.
1602 if (TrueIfSigned)
1603 return new ICmpInst(ICmpInst::ICMP_SGT, X,
1604 ConstantInt::getAllOnesValue(X->getType()));
1605 else
1606 return new ICmpInst(ICmpInst::ICMP_SLT, X,
1607 ConstantInt::getNullValue(X->getType()));
1608 }
1609
1610 if (Xor->hasOneUse()) {
1611 // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask))
1612 if (!Cmp.isEquality() && XorC->isSignMask()) {
1613 Pred = Cmp.getFlippedSignednessPredicate();
1614 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC));
1615 }
1616
1617 // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask))
1618 if (!Cmp.isEquality() && XorC->isMaxSignedValue()) {
1619 Pred = Cmp.getFlippedSignednessPredicate();
1620 Pred = Cmp.getSwappedPredicate(Pred);
1621 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC));
1622 }
1623 }
1624
1625 // Mask constant magic can eliminate an 'xor' with unsigned compares.
1626 if (Pred == ICmpInst::ICMP_UGT) {
1627 // (xor X, ~C) >u C --> X <u ~C (when C+1 is a power of 2)
1628 if (*XorC == ~C && (C + 1).isPowerOf2())
1629 return new ICmpInst(ICmpInst::ICMP_ULT, X, Y);
1630 // (xor X, C) >u C --> X >u C (when C+1 is a power of 2)
1631 if (*XorC == C && (C + 1).isPowerOf2())
1632 return new ICmpInst(ICmpInst::ICMP_UGT, X, Y);
1633 }
1634 if (Pred == ICmpInst::ICMP_ULT) {
1635 // (xor X, -C) <u C --> X >u ~C (when C is a power of 2)
1636 if (*XorC == -C && C.isPowerOf2())
1637 return new ICmpInst(ICmpInst::ICMP_UGT, X,
1638 ConstantInt::get(X->getType(), ~C));
1639 // (xor X, C) <u C --> X >u ~C (when -C is a power of 2)
1640 if (*XorC == C && (-C).isPowerOf2())
1641 return new ICmpInst(ICmpInst::ICMP_UGT, X,
1642 ConstantInt::get(X->getType(), ~C));
1643 }
1644 return nullptr;
1645}
1646
1647/// For power-of-2 C:
1648/// ((X s>> ShiftC) ^ X) u< C --> (X + C) u< (C << 1)
1649/// ((X s>> ShiftC) ^ X) u> (C - 1) --> (X + C) u> ((C << 1) - 1)
1652 const APInt &C) {
1653 CmpInst::Predicate Pred = Cmp.getPredicate();
1654 APInt PowerOf2;
1655 if (Pred == ICmpInst::ICMP_ULT)
1656 PowerOf2 = C;
1657 else if (Pred == ICmpInst::ICMP_UGT && !C.isMaxValue())
1658 PowerOf2 = C + 1;
1659 else
1660 return nullptr;
1661 if (!PowerOf2.isPowerOf2())
1662 return nullptr;
1663 Value *X;
1664 const APInt *ShiftC;
1666 m_AShr(m_Deferred(X), m_APInt(ShiftC))))))
1667 return nullptr;
1668 uint64_t Shift = ShiftC->getLimitedValue();
1669 Type *XType = X->getType();
1670 if (Shift == 0 || PowerOf2.isMinSignedValue())
1671 return nullptr;
1672 Value *Add = Builder.CreateAdd(X, ConstantInt::get(XType, PowerOf2));
1673 APInt Bound =
1674 Pred == ICmpInst::ICMP_ULT ? PowerOf2 << 1 : ((PowerOf2 << 1) - 1);
1675 return new ICmpInst(Pred, Add, ConstantInt::get(XType, Bound));
1676}
1677
1678/// Fold icmp (and (sh X, Y), C2), C1.
1681 const APInt &C1,
1682 const APInt &C2) {
1683 BinaryOperator *Shift = dyn_cast<BinaryOperator>(And->getOperand(0));
1684 if (!Shift || !Shift->isShift())
1685 return nullptr;
1686
1687 // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could
1688 // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in
1689 // code produced by the clang front-end, for bitfield access.
1690 // This seemingly simple opportunity to fold away a shift turns out to be
1691 // rather complicated. See PR17827 for details.
1692 unsigned ShiftOpcode = Shift->getOpcode();
1693 bool IsShl = ShiftOpcode == Instruction::Shl;
1694 const APInt *C3;
1695 if (match(Shift->getOperand(1), m_APInt(C3))) {
1696 APInt NewAndCst, NewCmpCst;
1697 bool AnyCmpCstBitsShiftedOut;
1698 if (ShiftOpcode == Instruction::Shl) {
1699 // For a left shift, we can fold if the comparison is not signed. We can
1700 // also fold a signed comparison if the mask value and comparison value
1701 // are not negative. These constraints may not be obvious, but we can
1702 // prove that they are correct using an SMT solver.
1703 if (Cmp.isSigned() && (C2.isNegative() || C1.isNegative()))
1704 return nullptr;
1705
1706 NewCmpCst = C1.lshr(*C3);
1707 NewAndCst = C2.lshr(*C3);
1708 AnyCmpCstBitsShiftedOut = NewCmpCst.shl(*C3) != C1;
1709 } else if (ShiftOpcode == Instruction::LShr) {
1710 // For a logical right shift, we can fold if the comparison is not signed.
1711 // We can also fold a signed comparison if the shifted mask value and the
1712 // shifted comparison value are not negative. These constraints may not be
1713 // obvious, but we can prove that they are correct using an SMT solver.
1714 NewCmpCst = C1.shl(*C3);
1715 NewAndCst = C2.shl(*C3);
1716 AnyCmpCstBitsShiftedOut = NewCmpCst.lshr(*C3) != C1;
1717 if (Cmp.isSigned() && (NewAndCst.isNegative() || NewCmpCst.isNegative()))
1718 return nullptr;
1719 } else {
1720 // For an arithmetic shift, check that both constants don't use (in a
1721 // signed sense) the top bits being shifted out.
1722 assert(ShiftOpcode == Instruction::AShr && "Unknown shift opcode");
1723 NewCmpCst = C1.shl(*C3);
1724 NewAndCst = C2.shl(*C3);
1725 AnyCmpCstBitsShiftedOut = NewCmpCst.ashr(*C3) != C1;
1726 if (NewAndCst.ashr(*C3) != C2)
1727 return nullptr;
1728 }
1729
1730 if (AnyCmpCstBitsShiftedOut) {
1731 // If we shifted bits out, the fold is not going to work out. As a
1732 // special case, check to see if this means that the result is always
1733 // true or false now.
1734 if (Cmp.getPredicate() == ICmpInst::ICMP_EQ)
1735 return replaceInstUsesWith(Cmp, ConstantInt::getFalse(Cmp.getType()));
1736 if (Cmp.getPredicate() == ICmpInst::ICMP_NE)
1737 return replaceInstUsesWith(Cmp, ConstantInt::getTrue(Cmp.getType()));
1738 } else {
1739 Value *NewAnd = Builder.CreateAnd(
1740 Shift->getOperand(0), ConstantInt::get(And->getType(), NewAndCst));
1741 return new ICmpInst(Cmp.getPredicate(),
1742 NewAnd, ConstantInt::get(And->getType(), NewCmpCst));
1743 }
1744 }
1745
1746 // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is
1747 // preferable because it allows the C2 << Y expression to be hoisted out of a
1748 // loop if Y is invariant and X is not.
1749 if (Shift->hasOneUse() && C1.isZero() && Cmp.isEquality() &&
1750 !Shift->isArithmeticShift() &&
1751 ((!IsShl && C2.isOne()) || !isa<Constant>(Shift->getOperand(0)))) {
1752 // Compute C2 << Y.
1753 Value *NewShift =
1754 IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1))
1755 : Builder.CreateShl(And->getOperand(1), Shift->getOperand(1));
1756
1757 // Compute X & (C2 << Y).
1758 Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift);
1759 return new ICmpInst(Cmp.getPredicate(), NewAnd, Cmp.getOperand(1));
1760 }
1761
1762 return nullptr;
1763}
1764
1765/// Fold icmp (and X, C2), C1.
1768 const APInt &C1) {
1769 bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE;
1770
1771 // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1
1772 // TODO: We canonicalize to the longer form for scalars because we have
1773 // better analysis/folds for icmp, and codegen may be better with icmp.
1774 if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isZero() &&
1775 match(And->getOperand(1), m_One()))
1776 return new TruncInst(And->getOperand(0), Cmp.getType());
1777
1778 const APInt *C2;
1779 Value *X;
1780 if (!match(And, m_And(m_Value(X), m_APInt(C2))))
1781 return nullptr;
1782
1783 // (and X, highmask) s> [0, ~highmask] --> X s> ~highmask
1784 if (Cmp.getPredicate() == ICmpInst::ICMP_SGT && C1.ule(~*C2) &&
1785 C2->isNegatedPowerOf2())
1786 return new ICmpInst(ICmpInst::ICMP_SGT, X,
1787 ConstantInt::get(X->getType(), ~*C2));
1788 // (and X, highmask) s< [1, -highmask] --> X s< -highmask
1789 if (Cmp.getPredicate() == ICmpInst::ICMP_SLT && !C1.isSignMask() &&
1790 (C1 - 1).ule(~*C2) && C2->isNegatedPowerOf2() && !C2->isSignMask())
1791 return new ICmpInst(ICmpInst::ICMP_SLT, X,
1792 ConstantInt::get(X->getType(), -*C2));
1793
1794 // Don't perform the following transforms if the AND has multiple uses
1795 if (!And->hasOneUse())
1796 return nullptr;
1797
1798 if (Cmp.isEquality() && C1.isZero()) {
1799 // Restrict this fold to single-use 'and' (PR10267).
1800 // Replace (and X, (1 << size(X)-1) != 0) with X s< 0
1801 if (C2->isSignMask()) {
1802 Constant *Zero = Constant::getNullValue(X->getType());
1803 auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE;
1804 return new ICmpInst(NewPred, X, Zero);
1805 }
1806
1807 APInt NewC2 = *C2;
1808 KnownBits Know = computeKnownBits(And->getOperand(0), 0, And);
1809 // Set high zeros of C2 to allow matching negated power-of-2.
1810 NewC2 = *C2 | APInt::getHighBitsSet(C2->getBitWidth(),
1811 Know.countMinLeadingZeros());
1812
1813 // Restrict this fold only for single-use 'and' (PR10267).
1814 // ((%x & C) == 0) --> %x u< (-C) iff (-C) is power of two.
1815 if (NewC2.isNegatedPowerOf2()) {
1816 Constant *NegBOC = ConstantInt::get(And->getType(), -NewC2);
1817 auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT;
1818 return new ICmpInst(NewPred, X, NegBOC);
1819 }
1820 }
1821
1822 // If the LHS is an 'and' of a truncate and we can widen the and/compare to
1823 // the input width without changing the value produced, eliminate the cast:
1824 //
1825 // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1'
1826 //
1827 // We can do this transformation if the constants do not have their sign bits
1828 // set or if it is an equality comparison. Extending a relational comparison
1829 // when we're checking the sign bit would not work.
1830 Value *W;
1831 if (match(And->getOperand(0), m_OneUse(m_Trunc(m_Value(W)))) &&
1832 (Cmp.isEquality() || (!C1.isNegative() && !C2->isNegative()))) {
1833 // TODO: Is this a good transform for vectors? Wider types may reduce
1834 // throughput. Should this transform be limited (even for scalars) by using
1835 // shouldChangeType()?
1836 if (!Cmp.getType()->isVectorTy()) {
1837 Type *WideType = W->getType();
1838 unsigned WideScalarBits = WideType->getScalarSizeInBits();
1839 Constant *ZextC1 = ConstantInt::get(WideType, C1.zext(WideScalarBits));
1840 Constant *ZextC2 = ConstantInt::get(WideType, C2->zext(WideScalarBits));
1841 Value *NewAnd = Builder.CreateAnd(W, ZextC2, And->getName());
1842 return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1);
1843 }
1844 }
1845
1846 if (Instruction *I = foldICmpAndShift(Cmp, And, C1, *C2))
1847 return I;
1848
1849 // (icmp pred (and (or (lshr A, B), A), 1), 0) -->
1850 // (icmp pred (and A, (or (shl 1, B), 1), 0))
1851 //
1852 // iff pred isn't signed
1853 if (!Cmp.isSigned() && C1.isZero() && And->getOperand(0)->hasOneUse() &&
1854 match(And->getOperand(1), m_One())) {
1855 Constant *One = cast<Constant>(And->getOperand(1));
1856 Value *Or = And->getOperand(0);
1857 Value *A, *B, *LShr;
1858 if (match(Or, m_Or(m_Value(LShr), m_Value(A))) &&
1859 match(LShr, m_LShr(m_Specific(A), m_Value(B)))) {
1860 unsigned UsesRemoved = 0;
1861 if (And->hasOneUse())
1862 ++UsesRemoved;
1863 if (Or->hasOneUse())
1864 ++UsesRemoved;
1865 if (LShr->hasOneUse())
1866 ++UsesRemoved;
1867
1868 // Compute A & ((1 << B) | 1)
1869 unsigned RequireUsesRemoved = match(B, m_ImmConstant()) ? 1 : 3;
1870 if (UsesRemoved >= RequireUsesRemoved) {
1871 Value *NewOr =
1872 Builder.CreateOr(Builder.CreateShl(One, B, LShr->getName(),
1873 /*HasNUW=*/true),
1874 One, Or->getName());
1875 Value *NewAnd = Builder.CreateAnd(A, NewOr, And->getName());
1876 return new ICmpInst(Cmp.getPredicate(), NewAnd, Cmp.getOperand(1));
1877 }
1878 }
1879 }
1880
1881 // (icmp eq (and (bitcast X to int), ExponentMask), ExponentMask) -->
1882 // llvm.is.fpclass(X, fcInf|fcNan)
1883 // (icmp ne (and (bitcast X to int), ExponentMask), ExponentMask) -->
1884 // llvm.is.fpclass(X, ~(fcInf|fcNan))
1885 Value *V;
1886 if (!Cmp.getParent()->getParent()->hasFnAttribute(
1887 Attribute::NoImplicitFloat) &&
1888 Cmp.isEquality() &&
1890 Type *FPType = V->getType()->getScalarType();
1891 if (FPType->isIEEELikeFPTy() && C1 == *C2) {
1892 APInt ExponentMask =
1893 APFloat::getInf(FPType->getFltSemantics()).bitcastToAPInt();
1894 if (C1 == ExponentMask) {
1895 unsigned Mask = FPClassTest::fcNan | FPClassTest::fcInf;
1896 if (isICMP_NE)
1897 Mask = ~Mask & fcAllFlags;
1898 return replaceInstUsesWith(Cmp, Builder.createIsFPClass(V, Mask));
1899 }
1900 }
1901 }
1902
1903 return nullptr;
1904}
1905
1906/// Fold icmp (and X, Y), C.
1909 const APInt &C) {
1910 if (Instruction *I = foldICmpAndConstConst(Cmp, And, C))
1911 return I;
1912
1913 const ICmpInst::Predicate Pred = Cmp.getPredicate();
1914 bool TrueIfNeg;
1915 if (isSignBitCheck(Pred, C, TrueIfNeg)) {
1916 // ((X - 1) & ~X) < 0 --> X == 0
1917 // ((X - 1) & ~X) >= 0 --> X != 0
1918 Value *X;
1919 if (match(And->getOperand(0), m_Add(m_Value(X), m_AllOnes())) &&
1920 match(And->getOperand(1), m_Not(m_Specific(X)))) {
1921 auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1922 return new ICmpInst(NewPred, X, ConstantInt::getNullValue(X->getType()));
1923 }
1924 // (X & -X) < 0 --> X == MinSignedC
1925 // (X & -X) > -1 --> X != MinSignedC
1926 if (match(And, m_c_And(m_Neg(m_Value(X)), m_Deferred(X)))) {
1927 Constant *MinSignedC = ConstantInt::get(
1928 X->getType(),
1929 APInt::getSignedMinValue(X->getType()->getScalarSizeInBits()));
1930 auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1931 return new ICmpInst(NewPred, X, MinSignedC);
1932 }
1933 }
1934
1935 // TODO: These all require that Y is constant too, so refactor with the above.
1936
1937 // Try to optimize things like "A[i] & 42 == 0" to index computations.
1938 Value *X = And->getOperand(0);
1939 Value *Y = And->getOperand(1);
1940 if (auto *C2 = dyn_cast<ConstantInt>(Y))
1941 if (auto *LI = dyn_cast<LoadInst>(X))
1942 if (auto *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0)))
1943 if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)))
1944 if (Instruction *Res =
1945 foldCmpLoadFromIndexedGlobal(LI, GEP, GV, Cmp, C2))
1946 return Res;
1947
1948 if (!Cmp.isEquality())
1949 return nullptr;
1950
1951 // X & -C == -C -> X > u ~C
1952 // X & -C != -C -> X <= u ~C
1953 // iff C is a power of 2
1954 if (Cmp.getOperand(1) == Y && C.isNegatedPowerOf2()) {
1955 auto NewPred =
1957 return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1))));
1958 }
1959
1960 // If we are testing the intersection of 2 select-of-nonzero-constants with no
1961 // common bits set, it's the same as checking if exactly one select condition
1962 // is set:
1963 // ((A ? TC : FC) & (B ? TC : FC)) == 0 --> xor A, B
1964 // ((A ? TC : FC) & (B ? TC : FC)) != 0 --> not(xor A, B)
1965 // TODO: Generalize for non-constant values.
1966 // TODO: Handle signed/unsigned predicates.
1967 // TODO: Handle other bitwise logic connectors.
1968 // TODO: Extend to handle a non-zero compare constant.
1969 if (C.isZero() && (Pred == CmpInst::ICMP_EQ || And->hasOneUse())) {
1970 assert(Cmp.isEquality() && "Not expecting non-equality predicates");
1971 Value *A, *B;
1972 const APInt *TC, *FC;
1973 if (match(X, m_Select(m_Value(A), m_APInt(TC), m_APInt(FC))) &&
1974 match(Y,
1975 m_Select(m_Value(B), m_SpecificInt(*TC), m_SpecificInt(*FC))) &&
1976 !TC->isZero() && !FC->isZero() && !TC->intersects(*FC)) {
1977 Value *R = Builder.CreateXor(A, B);
1978 if (Pred == CmpInst::ICMP_NE)
1979 R = Builder.CreateNot(R);
1980 return replaceInstUsesWith(Cmp, R);
1981 }
1982 }
1983
1984 // ((zext i1 X) & Y) == 0 --> !((trunc Y) & X)
1985 // ((zext i1 X) & Y) != 0 --> ((trunc Y) & X)
1986 // ((zext i1 X) & Y) == 1 --> ((trunc Y) & X)
1987 // ((zext i1 X) & Y) != 1 --> !((trunc Y) & X)
1989 X->getType()->isIntOrIntVectorTy(1) && (C.isZero() || C.isOne())) {
1990 Value *TruncY = Builder.CreateTrunc(Y, X->getType());
1991 if (C.isZero() ^ (Pred == CmpInst::ICMP_NE)) {
1992 Value *And = Builder.CreateAnd(TruncY, X);
1994 }
1995 return BinaryOperator::CreateAnd(TruncY, X);
1996 }
1997
1998 // (icmp eq/ne (and (shl -1, X), Y), 0)
1999 // -> (icmp eq/ne (lshr Y, X), 0)
2000 // We could technically handle any C == 0 or (C < 0 && isOdd(C)) but it seems
2001 // highly unlikely the non-zero case will ever show up in code.
2002 if (C.isZero() &&
2004 m_Value(Y))))) {
2005 Value *LShr = Builder.CreateLShr(Y, X);
2006 return new ICmpInst(Pred, LShr, Constant::getNullValue(LShr->getType()));
2007 }
2008
2009 // (icmp eq/ne (and (add A, Addend), Msk), C)
2010 // -> (icmp eq/ne (and A, Msk), (and (sub C, Addend), Msk))
2011 {
2012 Value *A;
2013 const APInt *Addend, *Msk;
2014 if (match(And, m_And(m_OneUse(m_Add(m_Value(A), m_APInt(Addend))),
2015 m_APInt(Msk))) &&
2016 Msk->isMask() && C.ule(*Msk)) {
2017 APInt NewComperand = (C - *Addend) & *Msk;
2018 Value* MaskA = Builder.CreateAnd(A, ConstantInt::get(A->getType(), *Msk));
2019 return new ICmpInst(
2020 Pred, MaskA,
2021 Constant::getIntegerValue(MaskA->getType(), NewComperand));
2022 }
2023 }
2024
2025 return nullptr;
2026}
2027
2028/// Fold icmp eq/ne (or (xor/sub (X1, X2), xor/sub (X3, X4))), 0.
2030 InstCombiner::BuilderTy &Builder) {
2031 // Are we using xors or subs to bitwise check for a pair or pairs of
2032 // (in)equalities? Convert to a shorter form that has more potential to be
2033 // folded even further.
2034 // ((X1 ^/- X2) || (X3 ^/- X4)) == 0 --> (X1 == X2) && (X3 == X4)
2035 // ((X1 ^/- X2) || (X3 ^/- X4)) != 0 --> (X1 != X2) || (X3 != X4)
2036 // ((X1 ^/- X2) || (X3 ^/- X4) || (X5 ^/- X6)) == 0 -->
2037 // (X1 == X2) && (X3 == X4) && (X5 == X6)
2038 // ((X1 ^/- X2) || (X3 ^/- X4) || (X5 ^/- X6)) != 0 -->
2039 // (X1 != X2) || (X3 != X4) || (X5 != X6)
2041 SmallVector<Value *, 16> WorkList(1, Or);
2042
2043 while (!WorkList.empty()) {
2044 auto MatchOrOperatorArgument = [&](Value *OrOperatorArgument) {
2045 Value *Lhs, *Rhs;
2046
2047 if (match(OrOperatorArgument,
2048 m_OneUse(m_Xor(m_Value(Lhs), m_Value(Rhs))))) {
2049 CmpValues.emplace_back(Lhs, Rhs);
2050 return;
2051 }
2052
2053 if (match(OrOperatorArgument,
2054 m_OneUse(m_Sub(m_Value(Lhs), m_Value(Rhs))))) {
2055 CmpValues.emplace_back(Lhs, Rhs);
2056 return;
2057 }
2058
2059 WorkList.push_back(OrOperatorArgument);
2060 };
2061
2062 Value *CurrentValue = WorkList.pop_back_val();
2063 Value *OrOperatorLhs, *OrOperatorRhs;
2064
2065 if (!match(CurrentValue,
2066 m_Or(m_Value(OrOperatorLhs), m_Value(OrOperatorRhs)))) {
2067 return nullptr;
2068 }
2069
2070 MatchOrOperatorArgument(OrOperatorRhs);
2071 MatchOrOperatorArgument(OrOperatorLhs);
2072 }
2073
2074 ICmpInst::Predicate Pred = Cmp.getPredicate();
2075 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2076 Value *LhsCmp = Builder.CreateICmp(Pred, CmpValues.rbegin()->first,
2077 CmpValues.rbegin()->second);
2078
2079 for (auto It = CmpValues.rbegin() + 1; It != CmpValues.rend(); ++It) {
2080 Value *RhsCmp = Builder.CreateICmp(Pred, It->first, It->second);
2081 LhsCmp = Builder.CreateBinOp(BOpc, LhsCmp, RhsCmp);
2082 }
2083
2084 return LhsCmp;
2085}
2086
2087/// Fold icmp (or X, Y), C.
2090 const APInt &C) {
2091 ICmpInst::Predicate Pred = Cmp.getPredicate();
2092 if (C.isOne()) {
2093 // icmp slt signum(V) 1 --> icmp slt V, 1
2094 Value *V = nullptr;
2095 if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V))))
2096 return new ICmpInst(ICmpInst::ICMP_SLT, V,
2097 ConstantInt::get(V->getType(), 1));
2098 }
2099
2100 Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1);
2101
2102 // (icmp eq/ne (or disjoint x, C0), C1)
2103 // -> (icmp eq/ne x, C0^C1)
2104 if (Cmp.isEquality() && match(OrOp1, m_ImmConstant()) &&
2105 cast<PossiblyDisjointInst>(Or)->isDisjoint()) {
2106 Value *NewC =
2107 Builder.CreateXor(OrOp1, ConstantInt::get(OrOp1->getType(), C));
2108 return new ICmpInst(Pred, OrOp0, NewC);
2109 }
2110
2111 const APInt *MaskC;
2112 if (match(OrOp1, m_APInt(MaskC)) && Cmp.isEquality()) {
2113 if (*MaskC == C && (C + 1).isPowerOf2()) {
2114 // X | C == C --> X <=u C
2115 // X | C != C --> X >u C
2116 // iff C+1 is a power of 2 (C is a bitmask of the low bits)
2118 return new ICmpInst(Pred, OrOp0, OrOp1);
2119 }
2120
2121 // More general: canonicalize 'equality with set bits mask' to
2122 // 'equality with clear bits mask'.
2123 // (X | MaskC) == C --> (X & ~MaskC) == C ^ MaskC
2124 // (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC
2125 if (Or->hasOneUse()) {
2126 Value *And = Builder.CreateAnd(OrOp0, ~(*MaskC));
2127 Constant *NewC = ConstantInt::get(Or->getType(), C ^ (*MaskC));
2128 return new ICmpInst(Pred, And, NewC);
2129 }
2130 }
2131
2132 // (X | (X-1)) s< 0 --> X s< 1
2133 // (X | (X-1)) s> -1 --> X s> 0
2134 Value *X;
2135 bool TrueIfSigned;
2136 if (isSignBitCheck(Pred, C, TrueIfSigned) &&
2138 auto NewPred = TrueIfSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGT;
2139 Constant *NewC = ConstantInt::get(X->getType(), TrueIfSigned ? 1 : 0);
2140 return new ICmpInst(NewPred, X, NewC);
2141 }
2142
2143 const APInt *OrC;
2144 // icmp(X | OrC, C) --> icmp(X, 0)
2145 if (C.isNonNegative() && match(Or, m_Or(m_Value(X), m_APInt(OrC)))) {
2146 switch (Pred) {
2147 // X | OrC s< C --> X s< 0 iff OrC s>= C s>= 0
2148 case ICmpInst::ICMP_SLT:
2149 // X | OrC s>= C --> X s>= 0 iff OrC s>= C s>= 0
2150 case ICmpInst::ICMP_SGE:
2151 if (OrC->sge(C))
2152 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
2153 break;
2154 // X | OrC s<= C --> X s< 0 iff OrC s> C s>= 0
2155 case ICmpInst::ICMP_SLE:
2156 // X | OrC s> C --> X s>= 0 iff OrC s> C s>= 0
2157 case ICmpInst::ICMP_SGT:
2158 if (OrC->sgt(C))
2160 ConstantInt::getNullValue(X->getType()));
2161 break;
2162 default:
2163 break;
2164 }
2165 }
2166
2167 if (!Cmp.isEquality() || !C.isZero() || !Or->hasOneUse())
2168 return nullptr;
2169
2170 Value *P, *Q;
2172 // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0
2173 // -> and (icmp eq P, null), (icmp eq Q, null).
2174 Value *CmpP =
2175 Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType()));
2176 Value *CmpQ =
2178 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2179 return BinaryOperator::Create(BOpc, CmpP, CmpQ);
2180 }
2181
2182 if (Value *V = foldICmpOrXorSubChain(Cmp, Or, Builder))
2183 return replaceInstUsesWith(Cmp, V);
2184
2185 return nullptr;
2186}
2187
2188/// Fold icmp (mul X, Y), C.
2191 const APInt &C) {
2192 ICmpInst::Predicate Pred = Cmp.getPredicate();
2193 Type *MulTy = Mul->getType();
2194 Value *X = Mul->getOperand(0);
2195
2196 // If there's no overflow:
2197 // X * X == 0 --> X == 0
2198 // X * X != 0 --> X != 0
2199 if (Cmp.isEquality() && C.isZero() && X == Mul->getOperand(1) &&
2200 (Mul->hasNoUnsignedWrap() || Mul->hasNoSignedWrap()))
2201 return new ICmpInst(Pred, X, ConstantInt::getNullValue(MulTy));
2202
2203 const APInt *MulC;
2204 if (!match(Mul->getOperand(1), m_APInt(MulC)))
2205 return nullptr;
2206
2207 // If this is a test of the sign bit and the multiply is sign-preserving with
2208 // a constant operand, use the multiply LHS operand instead:
2209 // (X * +MulC) < 0 --> X < 0
2210 // (X * -MulC) < 0 --> X > 0
2211 if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) {
2212 if (MulC->isNegative())
2213 Pred = ICmpInst::getSwappedPredicate(Pred);
2214 return new ICmpInst(Pred, X, ConstantInt::getNullValue(MulTy));
2215 }
2216
2217 if (MulC->isZero())
2218 return nullptr;
2219
2220 // If the multiply does not wrap or the constant is odd, try to divide the
2221 // compare constant by the multiplication factor.
2222 if (Cmp.isEquality()) {
2223 // (mul nsw X, MulC) eq/ne C --> X eq/ne C /s MulC
2224 if (Mul->hasNoSignedWrap() && C.srem(*MulC).isZero()) {
2225 Constant *NewC = ConstantInt::get(MulTy, C.sdiv(*MulC));
2226 return new ICmpInst(Pred, X, NewC);
2227 }
2228
2229 // C % MulC == 0 is weaker than we could use if MulC is odd because it
2230 // correct to transform if MulC * N == C including overflow. I.e with i8
2231 // (icmp eq (mul X, 5), 101) -> (icmp eq X, 225) but since 101 % 5 != 0, we
2232 // miss that case.
2233 if (C.urem(*MulC).isZero()) {
2234 // (mul nuw X, MulC) eq/ne C --> X eq/ne C /u MulC
2235 // (mul X, OddC) eq/ne N * C --> X eq/ne N
2236 if ((*MulC & 1).isOne() || Mul->hasNoUnsignedWrap()) {
2237 Constant *NewC = ConstantInt::get(MulTy, C.udiv(*MulC));
2238 return new ICmpInst(Pred, X, NewC);
2239 }
2240 }
2241 }
2242
2243 // With a matching no-overflow guarantee, fold the constants:
2244 // (X * MulC) < C --> X < (C / MulC)
2245 // (X * MulC) > C --> X > (C / MulC)
2246 // TODO: Assert that Pred is not equal to SGE, SLE, UGE, ULE?
2247 Constant *NewC = nullptr;
2248 if (Mul->hasNoSignedWrap() && ICmpInst::isSigned(Pred)) {
2249 // MININT / -1 --> overflow.
2250 if (C.isMinSignedValue() && MulC->isAllOnes())
2251 return nullptr;
2252 if (MulC->isNegative())
2253 Pred = ICmpInst::getSwappedPredicate(Pred);
2254
2255 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) {
2256 NewC = ConstantInt::get(
2258 } else {
2259 assert((Pred == ICmpInst::ICMP_SLE || Pred == ICmpInst::ICMP_SGT) &&
2260 "Unexpected predicate");
2261 NewC = ConstantInt::get(
2263 }
2264 } else if (Mul->hasNoUnsignedWrap() && ICmpInst::isUnsigned(Pred)) {
2265 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) {
2266 NewC = ConstantInt::get(
2268 } else {
2269 assert((Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
2270 "Unexpected predicate");
2271 NewC = ConstantInt::get(
2273 }
2274 }
2275
2276 return NewC ? new ICmpInst(Pred, X, NewC) : nullptr;
2277}
2278
2279/// Fold icmp (shl nuw C2, Y), C.
2281 const APInt &C) {
2282 Value *Y;
2283 const APInt *C2;
2284 if (!match(Shl, m_NUWShl(m_APInt(C2), m_Value(Y))))
2285 return nullptr;
2286
2287 Type *ShiftType = Shl->getType();
2288 unsigned TypeBits = C.getBitWidth();
2289 ICmpInst::Predicate Pred = Cmp.getPredicate();
2290 if (Cmp.isUnsigned()) {
2291 if (C2->isZero() || C2->ugt(C))
2292 return nullptr;
2293 APInt Div, Rem;
2294 APInt::udivrem(C, *C2, Div, Rem);
2295 bool CIsPowerOf2 = Rem.isZero() && Div.isPowerOf2();
2296
2297 // (1 << Y) pred C -> Y pred Log2(C)
2298 if (!CIsPowerOf2) {
2299 // (1 << Y) < 30 -> Y <= 4
2300 // (1 << Y) <= 30 -> Y <= 4
2301 // (1 << Y) >= 30 -> Y > 4
2302 // (1 << Y) > 30 -> Y > 4
2303 if (Pred == ICmpInst::ICMP_ULT)
2304 Pred = ICmpInst::ICMP_ULE;
2305 else if (Pred == ICmpInst::ICMP_UGE)
2306 Pred = ICmpInst::ICMP_UGT;
2307 }
2308
2309 unsigned CLog2 = Div.logBase2();
2310 return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2));
2311 } else if (Cmp.isSigned() && C2->isOne()) {
2312 Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1);
2313 // (1 << Y) > 0 -> Y != 31
2314 // (1 << Y) > C -> Y != 31 if C is negative.
2315 if (Pred == ICmpInst::ICMP_SGT && C.sle(0))
2316 return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne);
2317
2318 // (1 << Y) < 0 -> Y == 31
2319 // (1 << Y) < 1 -> Y == 31
2320 // (1 << Y) < C -> Y == 31 if C is negative and not signed min.
2321 // Exclude signed min by subtracting 1 and lower the upper bound to 0.
2322 if (Pred == ICmpInst::ICMP_SLT && (C-1).sle(0))
2323 return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne);
2324 }
2325
2326 return nullptr;
2327}
2328
2329/// Fold icmp (shl X, Y), C.
2331 BinaryOperator *Shl,
2332 const APInt &C) {
2333 const APInt *ShiftVal;
2334 if (Cmp.isEquality() && match(Shl->getOperand(0), m_APInt(ShiftVal)))
2335 return foldICmpShlConstConst(Cmp, Shl->getOperand(1), C, *ShiftVal);
2336
2337 ICmpInst::Predicate Pred = Cmp.getPredicate();
2338 // (icmp pred (shl nuw&nsw X, Y), Csle0)
2339 // -> (icmp pred X, Csle0)
2340 //
2341 // The idea is the nuw/nsw essentially freeze the sign bit for the shift op
2342 // so X's must be what is used.
2343 if (C.sle(0) && Shl->hasNoUnsignedWrap() && Shl->hasNoSignedWrap())
2344 return new ICmpInst(Pred, Shl->getOperand(0), Cmp.getOperand(1));
2345
2346 // (icmp eq/ne (shl nuw|nsw X, Y), 0)
2347 // -> (icmp eq/ne X, 0)
2348 if (ICmpInst::isEquality(Pred) && C.isZero() &&
2349 (Shl->hasNoUnsignedWrap() || Shl->hasNoSignedWrap()))
2350 return new ICmpInst(Pred, Shl->getOperand(0), Cmp.getOperand(1));
2351
2352 // (icmp slt (shl nsw X, Y), 0/1)
2353 // -> (icmp slt X, 0/1)
2354 // (icmp sgt (shl nsw X, Y), 0/-1)
2355 // -> (icmp sgt X, 0/-1)
2356 //
2357 // NB: sge/sle with a constant will canonicalize to sgt/slt.
2358 if (Shl->hasNoSignedWrap() &&
2359 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT))
2360 if (C.isZero() || (Pred == ICmpInst::ICMP_SGT ? C.isAllOnes() : C.isOne()))
2361 return new ICmpInst(Pred, Shl->getOperand(0), Cmp.getOperand(1));
2362
2363 const APInt *ShiftAmt;
2364 if (!match(Shl->getOperand(1), m_APInt(ShiftAmt)))
2365 return foldICmpShlLHSC(Cmp, Shl, C);
2366
2367 // Check that the shift amount is in range. If not, don't perform undefined
2368 // shifts. When the shift is visited, it will be simplified.
2369 unsigned TypeBits = C.getBitWidth();
2370 if (ShiftAmt->uge(TypeBits))
2371 return nullptr;
2372
2373 Value *X = Shl->getOperand(0);
2374 Type *ShType = Shl->getType();
2375
2376 // NSW guarantees that we are only shifting out sign bits from the high bits,
2377 // so we can ASHR the compare constant without needing a mask and eliminate
2378 // the shift.
2379 if (Shl->hasNoSignedWrap()) {
2380 if (Pred == ICmpInst::ICMP_SGT) {
2381 // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt)
2382 APInt ShiftedC = C.ashr(*ShiftAmt);
2383 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2384 }
2385 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2386 C.ashr(*ShiftAmt).shl(*ShiftAmt) == C) {
2387 APInt ShiftedC = C.ashr(*ShiftAmt);
2388 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2389 }
2390 if (Pred == ICmpInst::ICMP_SLT) {
2391 // SLE is the same as above, but SLE is canonicalized to SLT, so convert:
2392 // (X << S) <=s C is equiv to X <=s (C >> S) for all C
2393 // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX
2394 // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN
2395 assert(!C.isMinSignedValue() && "Unexpected icmp slt");
2396 APInt ShiftedC = (C - 1).ashr(*ShiftAmt) + 1;
2397 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2398 }
2399 }
2400
2401 // NUW guarantees that we are only shifting out zero bits from the high bits,
2402 // so we can LSHR the compare constant without needing a mask and eliminate
2403 // the shift.
2404 if (Shl->hasNoUnsignedWrap()) {
2405 if (Pred == ICmpInst::ICMP_UGT) {
2406 // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt)
2407 APInt ShiftedC = C.lshr(*ShiftAmt);
2408 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2409 }
2410 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2411 C.lshr(*ShiftAmt).shl(*ShiftAmt) == C) {
2412 APInt ShiftedC = C.lshr(*ShiftAmt);
2413 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2414 }
2415 if (Pred == ICmpInst::ICMP_ULT) {
2416 // ULE is the same as above, but ULE is canonicalized to ULT, so convert:
2417 // (X << S) <=u C is equiv to X <=u (C >> S) for all C
2418 // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u
2419 // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0
2420 assert(C.ugt(0) && "ult 0 should have been eliminated");
2421 APInt ShiftedC = (C - 1).lshr(*ShiftAmt) + 1;
2422 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2423 }
2424 }
2425
2426 if (Cmp.isEquality() && Shl->hasOneUse()) {
2427 // Strength-reduce the shift into an 'and'.
2428 Constant *Mask = ConstantInt::get(
2429 ShType,
2430 APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt->getZExtValue()));
2431 Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask");
2432 Constant *LShrC = ConstantInt::get(ShType, C.lshr(*ShiftAmt));
2433 return new ICmpInst(Pred, And, LShrC);
2434 }
2435
2436 // Otherwise, if this is a comparison of the sign bit, simplify to and/test.
2437 bool TrueIfSigned = false;
2438 if (Shl->hasOneUse() && isSignBitCheck(Pred, C, TrueIfSigned)) {
2439 // (X << 31) <s 0 --> (X & 1) != 0
2440 Constant *Mask = ConstantInt::get(
2441 ShType,
2442 APInt::getOneBitSet(TypeBits, TypeBits - ShiftAmt->getZExtValue() - 1));
2443 Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask");
2444 return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ,
2445 And, Constant::getNullValue(ShType));
2446 }
2447
2448 // Simplify 'shl' inequality test into 'and' equality test.
2449 if (Cmp.isUnsigned() && Shl->hasOneUse()) {
2450 // (X l<< C2) u<=/u> C1 iff C1+1 is power of two -> X & (~C1 l>> C2) ==/!= 0
2451 if ((C + 1).isPowerOf2() &&
2452 (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)) {
2453 Value *And = Builder.CreateAnd(X, (~C).lshr(ShiftAmt->getZExtValue()));
2454 return new ICmpInst(Pred == ICmpInst::ICMP_ULE ? ICmpInst::ICMP_EQ
2456 And, Constant::getNullValue(ShType));
2457 }
2458 // (X l<< C2) u</u>= C1 iff C1 is power of two -> X & (-C1 l>> C2) ==/!= 0
2459 if (C.isPowerOf2() &&
2460 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) {
2461 Value *And =
2462 Builder.CreateAnd(X, (~(C - 1)).lshr(ShiftAmt->getZExtValue()));
2463 return new ICmpInst(Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_EQ
2465 And, Constant::getNullValue(ShType));
2466 }
2467 }
2468
2469 // Transform (icmp pred iM (shl iM %v, N), C)
2470 // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N))
2471 // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N.
2472 // This enables us to get rid of the shift in favor of a trunc that may be
2473 // free on the target. It has the additional benefit of comparing to a
2474 // smaller constant that may be more target-friendly.
2475 unsigned Amt = ShiftAmt->getLimitedValue(TypeBits - 1);
2476 if (Shl->hasOneUse() && Amt != 0 &&
2477 shouldChangeType(ShType->getScalarSizeInBits(), TypeBits - Amt)) {
2478 ICmpInst::Predicate CmpPred = Pred;
2479 APInt RHSC = C;
2480
2481 if (RHSC.countr_zero() < Amt && ICmpInst::isStrictPredicate(CmpPred)) {
2482 // Try the flipped strictness predicate.
2483 // e.g.:
2484 // icmp ult i64 (shl X, 32), 8589934593 ->
2485 // icmp ule i64 (shl X, 32), 8589934592 ->
2486 // icmp ule i32 (trunc X, i32), 2 ->
2487 // icmp ult i32 (trunc X, i32), 3
2488 if (auto FlippedStrictness =
2490 Pred, ConstantInt::get(ShType->getContext(), C))) {
2491 CmpPred = FlippedStrictness->first;
2492 RHSC = cast<ConstantInt>(FlippedStrictness->second)->getValue();
2493 }
2494 }
2495
2496 if (RHSC.countr_zero() >= Amt) {
2497 Type *TruncTy = ShType->getWithNewBitWidth(TypeBits - Amt);
2498 Constant *NewC =
2499 ConstantInt::get(TruncTy, RHSC.ashr(*ShiftAmt).trunc(TypeBits - Amt));
2500 return new ICmpInst(CmpPred,
2501 Builder.CreateTrunc(X, TruncTy, "", /*IsNUW=*/false,
2502 Shl->hasNoSignedWrap()),
2503 NewC);
2504 }
2505 }
2506
2507 return nullptr;
2508}
2509
2510/// Fold icmp ({al}shr X, Y), C.
2512 BinaryOperator *Shr,
2513 const APInt &C) {
2514 // An exact shr only shifts out zero bits, so:
2515 // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0
2516 Value *X = Shr->getOperand(0);
2517 CmpInst::Predicate Pred = Cmp.getPredicate();
2518 if (Cmp.isEquality() && Shr->isExact() && C.isZero())
2519 return new ICmpInst(Pred, X, Cmp.getOperand(1));
2520
2521 bool IsAShr = Shr->getOpcode() == Instruction::AShr;
2522 const APInt *ShiftValC;
2523 if (match(X, m_APInt(ShiftValC))) {
2524 if (Cmp.isEquality())
2525 return foldICmpShrConstConst(Cmp, Shr->getOperand(1), C, *ShiftValC);
2526
2527 // (ShiftValC >> Y) >s -1 --> Y != 0 with ShiftValC < 0
2528 // (ShiftValC >> Y) <s 0 --> Y == 0 with ShiftValC < 0
2529 bool TrueIfSigned;
2530 if (!IsAShr && ShiftValC->isNegative() &&
2531 isSignBitCheck(Pred, C, TrueIfSigned))
2532 return new ICmpInst(TrueIfSigned ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE,
2533 Shr->getOperand(1),
2534 ConstantInt::getNullValue(X->getType()));
2535
2536 // If the shifted constant is a power-of-2, test the shift amount directly:
2537 // (ShiftValC >> Y) >u C --> X <u (LZ(C) - LZ(ShiftValC))
2538 // (ShiftValC >> Y) <u C --> X >=u (LZ(C-1) - LZ(ShiftValC))
2539 if (!IsAShr && ShiftValC->isPowerOf2() &&
2540 (Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_ULT)) {
2541 bool IsUGT = Pred == CmpInst::ICMP_UGT;
2542 assert(ShiftValC->uge(C) && "Expected simplify of compare");
2543 assert((IsUGT || !C.isZero()) && "Expected X u< 0 to simplify");
2544
2545 unsigned CmpLZ = IsUGT ? C.countl_zero() : (C - 1).countl_zero();
2546 unsigned ShiftLZ = ShiftValC->countl_zero();
2547 Constant *NewC = ConstantInt::get(Shr->getType(), CmpLZ - ShiftLZ);
2548 auto NewPred = IsUGT ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE;
2549 return new ICmpInst(NewPred, Shr->getOperand(1), NewC);
2550 }
2551 }
2552
2553 const APInt *ShiftAmtC;
2554 if (!match(Shr->getOperand(1), m_APInt(ShiftAmtC)))
2555 return nullptr;
2556
2557 // Check that the shift amount is in range. If not, don't perform undefined
2558 // shifts. When the shift is visited it will be simplified.
2559 unsigned TypeBits = C.getBitWidth();
2560 unsigned ShAmtVal = ShiftAmtC->getLimitedValue(TypeBits);
2561 if (ShAmtVal >= TypeBits || ShAmtVal == 0)
2562 return nullptr;
2563
2564 bool IsExact = Shr->isExact();
2565 Type *ShrTy = Shr->getType();
2566 // TODO: If we could guarantee that InstSimplify would handle all of the
2567 // constant-value-based preconditions in the folds below, then we could assert
2568 // those conditions rather than checking them. This is difficult because of
2569 // undef/poison (PR34838).
2570 if (IsAShr && Shr->hasOneUse()) {
2571 if (IsExact && (Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) &&
2572 (C - 1).isPowerOf2() && C.countLeadingZeros() > ShAmtVal) {
2573 // When C - 1 is a power of two and the transform can be legally
2574 // performed, prefer this form so the produced constant is close to a
2575 // power of two.
2576 // icmp slt/ult (ashr exact X, ShAmtC), C
2577 // --> icmp slt/ult X, (C - 1) << ShAmtC) + 1
2578 APInt ShiftedC = (C - 1).shl(ShAmtVal) + 1;
2579 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2580 }
2581 if (IsExact || Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) {
2582 // When ShAmtC can be shifted losslessly:
2583 // icmp PRED (ashr exact X, ShAmtC), C --> icmp PRED X, (C << ShAmtC)
2584 // icmp slt/ult (ashr X, ShAmtC), C --> icmp slt/ult X, (C << ShAmtC)
2585 APInt ShiftedC = C.shl(ShAmtVal);
2586 if (ShiftedC.ashr(ShAmtVal) == C)
2587 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2588 }
2589 if (Pred == CmpInst::ICMP_SGT) {
2590 // icmp sgt (ashr X, ShAmtC), C --> icmp sgt X, ((C + 1) << ShAmtC) - 1
2591 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2592 if (!C.isMaxSignedValue() && !(C + 1).shl(ShAmtVal).isMinSignedValue() &&
2593 (ShiftedC + 1).ashr(ShAmtVal) == (C + 1))
2594 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2595 }
2596 if (Pred == CmpInst::ICMP_UGT) {
2597 // icmp ugt (ashr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1
2598 // 'C + 1 << ShAmtC' can overflow as a signed number, so the 2nd
2599 // clause accounts for that pattern.
2600 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2601 if ((ShiftedC + 1).ashr(ShAmtVal) == (C + 1) ||
2602 (C + 1).shl(ShAmtVal).isMinSignedValue())
2603 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2604 }
2605
2606 // If the compare constant has significant bits above the lowest sign-bit,
2607 // then convert an unsigned cmp to a test of the sign-bit:
2608 // (ashr X, ShiftC) u> C --> X s< 0
2609 // (ashr X, ShiftC) u< C --> X s> -1
2610 if (C.getBitWidth() > 2 && C.getNumSignBits() <= ShAmtVal) {
2611 if (Pred == CmpInst::ICMP_UGT) {
2612 return new ICmpInst(CmpInst::ICMP_SLT, X,
2614 }
2615 if (Pred == CmpInst::ICMP_ULT) {
2616 return new ICmpInst(CmpInst::ICMP_SGT, X,
2618 }
2619 }
2620 } else if (!IsAShr) {
2621 if (Pred == CmpInst::ICMP_ULT || (Pred == CmpInst::ICMP_UGT && IsExact)) {
2622 // icmp ult (lshr X, ShAmtC), C --> icmp ult X, (C << ShAmtC)
2623 // icmp ugt (lshr exact X, ShAmtC), C --> icmp ugt X, (C << ShAmtC)
2624 APInt ShiftedC = C.shl(ShAmtVal);
2625 if (ShiftedC.lshr(ShAmtVal) == C)
2626 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2627 }
2628 if (Pred == CmpInst::ICMP_UGT) {
2629 // icmp ugt (lshr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1
2630 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2631 if ((ShiftedC + 1).lshr(ShAmtVal) == (C + 1))
2632 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2633 }
2634 }
2635
2636 if (!Cmp.isEquality())
2637 return nullptr;
2638
2639 // Handle equality comparisons of shift-by-constant.
2640
2641 // If the comparison constant changes with the shift, the comparison cannot
2642 // succeed (bits of the comparison constant cannot match the shifted value).
2643 // This should be known by InstSimplify and already be folded to true/false.
2644 assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) ||
2645 (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) &&
2646 "Expected icmp+shr simplify did not occur.");
2647
2648 // If the bits shifted out are known zero, compare the unshifted value:
2649 // (X & 4) >> 1 == 2 --> (X & 4) == 4.
2650 if (Shr->isExact())
2651 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, C << ShAmtVal));
2652
2653 if (C.isZero()) {
2654 // == 0 is u< 1.
2655 if (Pred == CmpInst::ICMP_EQ)
2656 return new ICmpInst(CmpInst::ICMP_ULT, X,
2657 ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal)));
2658 else
2659 return new ICmpInst(CmpInst::ICMP_UGT, X,
2660 ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal) - 1));
2661 }
2662
2663 if (Shr->hasOneUse()) {
2664 // Canonicalize the shift into an 'and':
2665 // icmp eq/ne (shr X, ShAmt), C --> icmp eq/ne (and X, HiMask), (C << ShAmt)
2666 APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal));
2667 Constant *Mask = ConstantInt::get(ShrTy, Val);
2668 Value *And = Builder.CreateAnd(X, Mask, Shr->getName() + ".mask");
2669 return new ICmpInst(Pred, And, ConstantInt::get(ShrTy, C << ShAmtVal));
2670 }
2671
2672 return nullptr;
2673}
2674
2676 BinaryOperator *SRem,
2677 const APInt &C) {
2678 // Match an 'is positive' or 'is negative' comparison of remainder by a
2679 // constant power-of-2 value:
2680 // (X % pow2C) sgt/slt 0
2681 const ICmpInst::Predicate Pred = Cmp.getPredicate();
2682 if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_SLT &&
2683 Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
2684 return nullptr;
2685
2686 // TODO: The one-use check is standard because we do not typically want to
2687 // create longer instruction sequences, but this might be a special-case
2688 // because srem is not good for analysis or codegen.
2689 if (!SRem->hasOneUse())
2690 return nullptr;
2691
2692 const APInt *DivisorC;
2693 if (!match(SRem->getOperand(1), m_Power2(DivisorC)))
2694 return nullptr;
2695
2696 // For cmp_sgt/cmp_slt only zero valued C is handled.
2697 // For cmp_eq/cmp_ne only positive valued C is handled.
2698 if (((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT) &&
2699 !C.isZero()) ||
2700 ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2701 !C.isStrictlyPositive()))
2702 return nullptr;
2703
2704 // Mask off the sign bit and the modulo bits (low-bits).
2705 Type *Ty = SRem->getType();
2707 Constant *MaskC = ConstantInt::get(Ty, SignMask | (*DivisorC - 1));
2708 Value *And = Builder.CreateAnd(SRem->getOperand(0), MaskC);
2709
2710 if (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE)
2711 return new ICmpInst(Pred, And, ConstantInt::get(Ty, C));
2712
2713 // For 'is positive?' check that the sign-bit is clear and at least 1 masked
2714 // bit is set. Example:
2715 // (i8 X % 32) s> 0 --> (X & 159) s> 0
2716 if (Pred == ICmpInst::ICMP_SGT)
2718
2719 // For 'is negative?' check that the sign-bit is set and at least 1 masked
2720 // bit is set. Example:
2721 // (i16 X % 4) s< 0 --> (X & 32771) u> 32768
2722 return new ICmpInst(ICmpInst::ICMP_UGT, And, ConstantInt::get(Ty, SignMask));
2723}
2724
2725/// Fold icmp (udiv X, Y), C.
2727 BinaryOperator *UDiv,
2728 const APInt &C) {
2729 ICmpInst::Predicate Pred = Cmp.getPredicate();
2730 Value *X = UDiv->getOperand(0);
2731 Value *Y = UDiv->getOperand(1);
2732 Type *Ty = UDiv->getType();
2733
2734 const APInt *C2;
2735 if (!match(X, m_APInt(C2)))
2736 return nullptr;
2737
2738 assert(*C2 != 0 && "udiv 0, X should have been simplified already.");
2739
2740 // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1))
2741 if (Pred == ICmpInst::ICMP_UGT) {
2742 assert(!C.isMaxValue() &&
2743 "icmp ugt X, UINT_MAX should have been simplified already.");
2744 return new ICmpInst(ICmpInst::ICMP_ULE, Y,
2745 ConstantInt::get(Ty, C2->udiv(C + 1)));
2746 }
2747
2748 // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C)
2749 if (Pred == ICmpInst::ICMP_ULT) {
2750 assert(C != 0 && "icmp ult X, 0 should have been simplified already.");
2751 return new ICmpInst(ICmpInst::ICMP_UGT, Y,
2752 ConstantInt::get(Ty, C2->udiv(C)));
2753 }
2754
2755 return nullptr;
2756}
2757
2758/// Fold icmp ({su}div X, Y), C.
2760 BinaryOperator *Div,
2761 const APInt &C) {
2762 ICmpInst::Predicate Pred = Cmp.getPredicate();
2763 Value *X = Div->getOperand(0);
2764 Value *Y = Div->getOperand(1);
2765 Type *Ty = Div->getType();
2766 bool DivIsSigned = Div->getOpcode() == Instruction::SDiv;
2767
2768 // If unsigned division and the compare constant is bigger than
2769 // UMAX/2 (negative), there's only one pair of values that satisfies an
2770 // equality check, so eliminate the division:
2771 // (X u/ Y) == C --> (X == C) && (Y == 1)
2772 // (X u/ Y) != C --> (X != C) || (Y != 1)
2773 // Similarly, if signed division and the compare constant is exactly SMIN:
2774 // (X s/ Y) == SMIN --> (X == SMIN) && (Y == 1)
2775 // (X s/ Y) != SMIN --> (X != SMIN) || (Y != 1)
2776 if (Cmp.isEquality() && Div->hasOneUse() && C.isSignBitSet() &&
2777 (!DivIsSigned || C.isMinSignedValue())) {
2778 Value *XBig = Builder.CreateICmp(Pred, X, ConstantInt::get(Ty, C));
2779 Value *YOne = Builder.CreateICmp(Pred, Y, ConstantInt::get(Ty, 1));
2780 auto Logic = Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2781 return BinaryOperator::Create(Logic, XBig, YOne);
2782 }
2783
2784 // Fold: icmp pred ([us]div X, C2), C -> range test
2785 // Fold this div into the comparison, producing a range check.
2786 // Determine, based on the divide type, what the range is being
2787 // checked. If there is an overflow on the low or high side, remember
2788 // it, otherwise compute the range [low, hi) bounding the new value.
2789 // See: InsertRangeTest above for the kinds of replacements possible.
2790 const APInt *C2;
2791 if (!match(Y, m_APInt(C2)))
2792 return nullptr;
2793
2794 // FIXME: If the operand types don't match the type of the divide
2795 // then don't attempt this transform. The code below doesn't have the
2796 // logic to deal with a signed divide and an unsigned compare (and
2797 // vice versa). This is because (x /s C2) <s C produces different
2798 // results than (x /s C2) <u C or (x /u C2) <s C or even
2799 // (x /u C2) <u C. Simply casting the operands and result won't
2800 // work. :( The if statement below tests that condition and bails
2801 // if it finds it.
2802 if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned())
2803 return nullptr;
2804
2805 // The ProdOV computation fails on divide by 0 and divide by -1. Cases with
2806 // INT_MIN will also fail if the divisor is 1. Although folds of all these
2807 // division-by-constant cases should be present, we can not assert that they
2808 // have happened before we reach this icmp instruction.
2809 if (C2->isZero() || C2->isOne() || (DivIsSigned && C2->isAllOnes()))
2810 return nullptr;
2811
2812 // Compute Prod = C * C2. We are essentially solving an equation of
2813 // form X / C2 = C. We solve for X by multiplying C2 and C.
2814 // By solving for X, we can turn this into a range check instead of computing
2815 // a divide.
2816 APInt Prod = C * *C2;
2817
2818 // Determine if the product overflows by seeing if the product is not equal to
2819 // the divide. Make sure we do the same kind of divide as in the LHS
2820 // instruction that we're folding.
2821 bool ProdOV = (DivIsSigned ? Prod.sdiv(*C2) : Prod.udiv(*C2)) != C;
2822
2823 // If the division is known to be exact, then there is no remainder from the
2824 // divide, so the covered range size is unit, otherwise it is the divisor.
2825 APInt RangeSize = Div->isExact() ? APInt(C2->getBitWidth(), 1) : *C2;
2826
2827 // Figure out the interval that is being checked. For example, a comparison
2828 // like "X /u 5 == 0" is really checking that X is in the interval [0, 5).
2829 // Compute this interval based on the constants involved and the signedness of
2830 // the compare/divide. This computes a half-open interval, keeping track of
2831 // whether either value in the interval overflows. After analysis each
2832 // overflow variable is set to 0 if it's corresponding bound variable is valid
2833 // -1 if overflowed off the bottom end, or +1 if overflowed off the top end.
2834 int LoOverflow = 0, HiOverflow = 0;
2835 APInt LoBound, HiBound;
2836
2837 if (!DivIsSigned) { // udiv
2838 // e.g. X/5 op 3 --> [15, 20)
2839 LoBound = Prod;
2840 HiOverflow = LoOverflow = ProdOV;
2841 if (!HiOverflow) {
2842 // If this is not an exact divide, then many values in the range collapse
2843 // to the same result value.
2844 HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false);
2845 }
2846 } else if (C2->isStrictlyPositive()) { // Divisor is > 0.
2847 if (C.isZero()) { // (X / pos) op 0
2848 // Can't overflow. e.g. X/2 op 0 --> [-1, 2)
2849 LoBound = -(RangeSize - 1);
2850 HiBound = RangeSize;
2851 } else if (C.isStrictlyPositive()) { // (X / pos) op pos
2852 LoBound = Prod; // e.g. X/5 op 3 --> [15, 20)
2853 HiOverflow = LoOverflow = ProdOV;
2854 if (!HiOverflow)
2855 HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true);
2856 } else { // (X / pos) op neg
2857 // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14)
2858 HiBound = Prod + 1;
2859 LoOverflow = HiOverflow = ProdOV ? -1 : 0;
2860 if (!LoOverflow) {
2861 APInt DivNeg = -RangeSize;
2862 LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0;
2863 }
2864 }
2865 } else if (C2->isNegative()) { // Divisor is < 0.
2866 if (Div->isExact())
2867 RangeSize.negate();
2868 if (C.isZero()) { // (X / neg) op 0
2869 // e.g. X/-5 op 0 --> [-4, 5)
2870 LoBound = RangeSize + 1;
2871 HiBound = -RangeSize;
2872 if (HiBound == *C2) { // -INTMIN = INTMIN
2873 HiOverflow = 1; // [INTMIN+1, overflow)
2874 HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN
2875 }
2876 } else if (C.isStrictlyPositive()) { // (X / neg) op pos
2877 // e.g. X/-5 op 3 --> [-19, -14)
2878 HiBound = Prod + 1;
2879 HiOverflow = LoOverflow = ProdOV ? -1 : 0;
2880 if (!LoOverflow)
2881 LoOverflow =
2882 addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1 : 0;
2883 } else { // (X / neg) op neg
2884 LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20)
2885 LoOverflow = HiOverflow = ProdOV;
2886 if (!HiOverflow)
2887 HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true);
2888 }
2889
2890 // Dividing by a negative swaps the condition. LT <-> GT
2891 Pred = ICmpInst::getSwappedPredicate(Pred);
2892 }
2893
2894 switch (Pred) {
2895 default:
2896 llvm_unreachable("Unhandled icmp predicate!");
2897 case ICmpInst::ICMP_EQ:
2898 if (LoOverflow && HiOverflow)
2899 return replaceInstUsesWith(Cmp, Builder.getFalse());
2900 if (HiOverflow)
2901 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
2902 X, ConstantInt::get(Ty, LoBound));
2903 if (LoOverflow)
2904 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
2905 X, ConstantInt::get(Ty, HiBound));
2906 return replaceInstUsesWith(
2907 Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, true));
2908 case ICmpInst::ICMP_NE:
2909 if (LoOverflow && HiOverflow)
2910 return replaceInstUsesWith(Cmp, Builder.getTrue());
2911 if (HiOverflow)
2912 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
2913 X, ConstantInt::get(Ty, LoBound));
2914 if (LoOverflow)
2915 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
2916 X, ConstantInt::get(Ty, HiBound));
2917 return replaceInstUsesWith(
2918 Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, false));
2919 case ICmpInst::ICMP_ULT:
2920 case ICmpInst::ICMP_SLT:
2921 if (LoOverflow == +1) // Low bound is greater than input range.
2922 return replaceInstUsesWith(Cmp, Builder.getTrue());
2923 if (LoOverflow == -1) // Low bound is less than input range.
2924 return replaceInstUsesWith(Cmp, Builder.getFalse());
2925 return new ICmpInst(Pred, X, ConstantInt::get(Ty, LoBound));
2926 case ICmpInst::ICMP_UGT:
2927 case ICmpInst::ICMP_SGT:
2928 if (HiOverflow == +1) // High bound greater than input range.
2929 return replaceInstUsesWith(Cmp, Builder.getFalse());
2930 if (HiOverflow == -1) // High bound less than input range.
2931 return replaceInstUsesWith(Cmp, Builder.getTrue());
2932 if (Pred == ICmpInst::ICMP_UGT)
2933 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, HiBound));
2934 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, HiBound));
2935 }
2936
2937 return nullptr;
2938}
2939
2940/// Fold icmp (sub X, Y), C.
2942 BinaryOperator *Sub,
2943 const APInt &C) {
2944 Value *X = Sub->getOperand(0), *Y = Sub->getOperand(1);
2945 ICmpInst::Predicate Pred = Cmp.getPredicate();
2946 Type *Ty = Sub->getType();
2947
2948 // (SubC - Y) == C) --> Y == (SubC - C)
2949 // (SubC - Y) != C) --> Y != (SubC - C)
2950 Constant *SubC;
2951 if (Cmp.isEquality() && match(X, m_ImmConstant(SubC))) {
2952 return new ICmpInst(Pred, Y,
2953 ConstantExpr::getSub(SubC, ConstantInt::get(Ty, C)));
2954 }
2955
2956 // (icmp P (sub nuw|nsw C2, Y), C) -> (icmp swap(P) Y, C2-C)
2957 const APInt *C2;
2958 APInt SubResult;
2959 ICmpInst::Predicate SwappedPred = Cmp.getSwappedPredicate();
2960 bool HasNSW = Sub->hasNoSignedWrap();
2961 bool HasNUW = Sub->hasNoUnsignedWrap();
2962 if (match(X, m_APInt(C2)) &&
2963 ((Cmp.isUnsigned() && HasNUW) || (Cmp.isSigned() && HasNSW)) &&
2964 !subWithOverflow(SubResult, *C2, C, Cmp.isSigned()))
2965 return new ICmpInst(SwappedPred, Y, ConstantInt::get(Ty, SubResult));
2966
2967 // X - Y == 0 --> X == Y.
2968 // X - Y != 0 --> X != Y.
2969 // TODO: We allow this with multiple uses as long as the other uses are not
2970 // in phis. The phi use check is guarding against a codegen regression
2971 // for a loop test. If the backend could undo this (and possibly
2972 // subsequent transforms), we would not need this hack.
2973 if (Cmp.isEquality() && C.isZero() &&
2974 none_of((Sub->users()), [](const User *U) { return isa<PHINode>(U); }))
2975 return new ICmpInst(Pred, X, Y);
2976
2977 // The following transforms are only worth it if the only user of the subtract
2978 // is the icmp.
2979 // TODO: This is an artificial restriction for all of the transforms below
2980 // that only need a single replacement icmp. Can these use the phi test
2981 // like the transform above here?
2982 if (!Sub->hasOneUse())
2983 return nullptr;
2984
2985 if (Sub->hasNoSignedWrap()) {
2986 // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y)
2987 if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
2988 return new ICmpInst(ICmpInst::ICMP_SGE, X, Y);
2989
2990 // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y)
2991 if (Pred == ICmpInst::ICMP_SGT && C.isZero())
2992 return new ICmpInst(ICmpInst::ICMP_SGT, X, Y);
2993
2994 // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y)
2995 if (Pred == ICmpInst::ICMP_SLT && C.isZero())
2996 return new ICmpInst(ICmpInst::ICMP_SLT, X, Y);
2997
2998 // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y)
2999 if (Pred == ICmpInst::ICMP_SLT && C.isOne())
3000 return new ICmpInst(ICmpInst::ICMP_SLE, X, Y);
3001 }
3002
3003 if (!match(X, m_APInt(C2)))
3004 return nullptr;
3005
3006 // C2 - Y <u C -> (Y | (C - 1)) == C2
3007 // iff (C2 & (C - 1)) == C - 1 and C is a power of 2
3008 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() &&
3009 (*C2 & (C - 1)) == (C - 1))
3010 return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(Y, C - 1), X);
3011
3012 // C2 - Y >u C -> (Y | C) != C2
3013 // iff C2 & C == C and C + 1 is a power of 2
3014 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == C)
3015 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(Y, C), X);
3016
3017 // We have handled special cases that reduce.
3018 // Canonicalize any remaining sub to add as:
3019 // (C2 - Y) > C --> (Y + ~C2) < ~C
3020 Value *Add = Builder.CreateAdd(Y, ConstantInt::get(Ty, ~(*C2)), "notsub",
3021 HasNUW, HasNSW);
3022 return new ICmpInst(SwappedPred, Add, ConstantInt::get(Ty, ~C));
3023}
3024
3025static Value *createLogicFromTable(const std::bitset<4> &Table, Value *Op0,
3026 Value *Op1, IRBuilderBase &Builder,
3027 bool HasOneUse) {
3028 auto FoldConstant = [&](bool Val) {
3029 Constant *Res = Val ? Builder.getTrue() : Builder.getFalse();
3030 if (Op0->getType()->isVectorTy())
3032 cast<VectorType>(Op0->getType())->getElementCount(), Res);
3033 return Res;
3034 };
3035
3036 switch (Table.to_ulong()) {
3037 case 0: // 0 0 0 0
3038 return FoldConstant(false);
3039 case 1: // 0 0 0 1
3040 return HasOneUse ? Builder.CreateNot(Builder.CreateOr(Op0, Op1)) : nullptr;
3041 case 2: // 0 0 1 0
3042 return HasOneUse ? Builder.CreateAnd(Builder.CreateNot(Op0), Op1) : nullptr;
3043 case 3: // 0 0 1 1
3044 return Builder.CreateNot(Op0);
3045 case 4: // 0 1 0 0
3046 return HasOneUse ? Builder.CreateAnd(Op0, Builder.CreateNot(Op1)) : nullptr;
3047 case 5: // 0 1 0 1
3048 return Builder.CreateNot(Op1);
3049 case 6: // 0 1 1 0
3050 return Builder.CreateXor(Op0, Op1);
3051 case 7: // 0 1 1 1
3052 return HasOneUse ? Builder.CreateNot(Builder.CreateAnd(Op0, Op1)) : nullptr;
3053 case 8: // 1 0 0 0
3054 return Builder.CreateAnd(Op0, Op1);
3055 case 9: // 1 0 0 1
3056 return HasOneUse ? Builder.CreateNot(Builder.CreateXor(Op0, Op1)) : nullptr;
3057 case 10: // 1 0 1 0
3058 return Op1;
3059 case 11: // 1 0 1 1
3060 return HasOneUse ? Builder.CreateOr(Builder.CreateNot(Op0), Op1) : nullptr;
3061 case 12: // 1 1 0 0
3062 return Op0;
3063 case 13: // 1 1 0 1
3064 return HasOneUse ? Builder.CreateOr(Op0, Builder.CreateNot(Op1)) : nullptr;
3065 case 14: // 1 1 1 0
3066 return Builder.CreateOr(Op0, Op1);
3067 case 15: // 1 1 1 1
3068 return FoldConstant(true);
3069 default:
3070 llvm_unreachable("Invalid Operation");
3071 }
3072 return nullptr;
3073}
3074
3075/// Fold icmp (add X, Y), C.
3078 const APInt &C) {
3079 Value *Y = Add->getOperand(1);
3080 Value *X = Add->getOperand(0);
3081
3082 Value *Op0, *Op1;
3083 Instruction *Ext0, *Ext1;
3084 const CmpInst::Predicate Pred = Cmp.getPredicate();
3085 if (match(Add,
3088 m_ZExtOrSExt(m_Value(Op1))))) &&
3089 Op0->getType()->isIntOrIntVectorTy(1) &&
3090 Op1->getType()->isIntOrIntVectorTy(1)) {
3091 unsigned BW = C.getBitWidth();
3092 std::bitset<4> Table;
3093 auto ComputeTable = [&](bool Op0Val, bool Op1Val) {
3094 int Res = 0;
3095 if (Op0Val)
3096 Res += isa<ZExtInst>(Ext0) ? 1 : -1;
3097 if (Op1Val)
3098 Res += isa<ZExtInst>(Ext1) ? 1 : -1;
3099 return ICmpInst::compare(APInt(BW, Res, true), C, Pred);
3100 };
3101
3102 Table[0] = ComputeTable(false, false);
3103 Table[1] = ComputeTable(false, true);
3104 Table[2] = ComputeTable(true, false);
3105 Table[3] = ComputeTable(true, true);
3106 if (auto *Cond =
3107 createLogicFromTable(Table, Op0, Op1, Builder, Add->hasOneUse()))
3108 return replaceInstUsesWith(Cmp, Cond);
3109 }
3110 const APInt *C2;
3111 if (Cmp.isEquality() || !match(Y, m_APInt(C2)))
3112 return nullptr;
3113
3114 // Fold icmp pred (add X, C2), C.
3115 Type *Ty = Add->getType();
3116
3117 // If the add does not wrap, we can always adjust the compare by subtracting
3118 // the constants. Equality comparisons are handled elsewhere. SGE/SLE/UGE/ULE
3119 // are canonicalized to SGT/SLT/UGT/ULT.
3120 if ((Add->hasNoSignedWrap() &&
3121 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) ||
3122 (Add->hasNoUnsignedWrap() &&
3123 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT))) {
3124 bool Overflow;
3125 APInt NewC =
3126 Cmp.isSigned() ? C.ssub_ov(*C2, Overflow) : C.usub_ov(*C2, Overflow);
3127 // If there is overflow, the result must be true or false.
3128 // TODO: Can we assert there is no overflow because InstSimplify always
3129 // handles those cases?
3130 if (!Overflow)
3131 // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2)
3132 return new ICmpInst(Pred, X, ConstantInt::get(Ty, NewC));
3133 }
3134
3135 if (ICmpInst::isUnsigned(Pred) && Add->hasNoSignedWrap() &&
3136 C.isNonNegative() && (C - *C2).isNonNegative() &&
3137 computeConstantRange(X, /*ForSigned=*/true).add(*C2).isAllNonNegative())
3138 return new ICmpInst(ICmpInst::getSignedPredicate(Pred), X,
3139 ConstantInt::get(Ty, C - *C2));
3140
3141 auto CR = ConstantRange::makeExactICmpRegion(Pred, C).subtract(*C2);
3142 const APInt &Upper = CR.getUpper();
3143 const APInt &Lower = CR.getLower();
3144 if (Cmp.isSigned()) {
3145 if (Lower.isSignMask())
3146 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, Upper));
3147 if (Upper.isSignMask())
3148 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, Lower));
3149 } else {
3150 if (Lower.isMinValue())
3151 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, Upper));
3152 if (Upper.isMinValue())
3153 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, Lower));
3154 }
3155
3156 // This set of folds is intentionally placed after folds that use no-wrapping
3157 // flags because those folds are likely better for later analysis/codegen.
3160
3161 // Fold compare with offset to opposite sign compare if it eliminates offset:
3162 // (X + C2) >u C --> X <s -C2 (if C == C2 + SMAX)
3163 if (Pred == CmpInst::ICMP_UGT && C == *C2 + SMax)
3164 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, -(*C2)));
3165
3166 // (X + C2) <u C --> X >s ~C2 (if C == C2 + SMIN)
3167 if (Pred == CmpInst::ICMP_ULT && C == *C2 + SMin)
3168 return new ICmpInst(ICmpInst::ICMP_SGT, X, ConstantInt::get(Ty, ~(*C2)));
3169
3170 // (X + C2) >s C --> X <u (SMAX - C) (if C == C2 - 1)
3171 if (Pred == CmpInst::ICMP_SGT && C == *C2 - 1)
3172 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, SMax - C));
3173
3174 // (X + C2) <s C --> X >u (C ^ SMAX) (if C == C2)
3175 if (Pred == CmpInst::ICMP_SLT && C == *C2)
3176 return new ICmpInst(ICmpInst::ICMP_UGT, X, ConstantInt::get(Ty, C ^ SMax));
3177
3178 // (X + -1) <u C --> X <=u C (if X is never null)
3179 if (Pred == CmpInst::ICMP_ULT && C2->isAllOnes()) {
3180 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
3181 if (llvm::isKnownNonZero(X, Q))
3182 return new ICmpInst(ICmpInst::ICMP_ULE, X, ConstantInt::get(Ty, C));
3183 }
3184
3185 if (!Add->hasOneUse())
3186 return nullptr;
3187
3188 // X+C <u C2 -> (X & -C2) == C
3189 // iff C & (C2-1) == 0
3190 // C2 is a power of 2
3191 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && (*C2 & (C - 1)) == 0)
3193 ConstantExpr::getNeg(cast<Constant>(Y)));
3194
3195 // X+C2 <u C -> (X & C) == 2C
3196 // iff C == -(C2)
3197 // C2 is a power of 2
3198 if (Pred == ICmpInst::ICMP_ULT && C2->isPowerOf2() && C == -*C2)
3200 ConstantInt::get(Ty, C * 2));
3201
3202 // X+C >u C2 -> (X & ~C2) != C
3203 // iff C & C2 == 0
3204 // C2+1 is a power of 2
3205 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == 0)
3207 ConstantExpr::getNeg(cast<Constant>(Y)));
3208
3209 // The range test idiom can use either ult or ugt. Arbitrarily canonicalize
3210 // to the ult form.
3211 // X+C2 >u C -> X+(C2-C-1) <u ~C
3212 if (Pred == ICmpInst::ICMP_UGT)
3213 return new ICmpInst(ICmpInst::ICMP_ULT,
3214 Builder.CreateAdd(X, ConstantInt::get(Ty, *C2 - C - 1)),
3215 ConstantInt::get(Ty, ~C));
3216
3217 // zext(V) + C2 pred C -> V + C3 pred' C4
3218 Value *V;
3219 if (match(X, m_ZExt(m_Value(V)))) {
3220 Type *NewCmpTy = V->getType();
3221 unsigned NewCmpBW = NewCmpTy->getScalarSizeInBits();
3222 if (shouldChangeType(Ty, NewCmpTy)) {
3223 if (CR.getActiveBits() <= NewCmpBW) {
3224 ConstantRange SrcCR = CR.truncate(NewCmpBW);
3225 CmpInst::Predicate EquivPred;
3226 APInt EquivInt;
3227 APInt EquivOffset;
3228
3229 SrcCR.getEquivalentICmp(EquivPred, EquivInt, EquivOffset);
3230 return new ICmpInst(
3231 EquivPred,
3232 EquivOffset.isZero()
3233 ? V
3234 : Builder.CreateAdd(V, ConstantInt::get(NewCmpTy, EquivOffset)),
3235 ConstantInt::get(NewCmpTy, EquivInt));
3236 }
3237 }
3238 }
3239
3240 return nullptr;
3241}
3242
3244 Value *&RHS, ConstantInt *&Less,
3245 ConstantInt *&Equal,
3246 ConstantInt *&Greater) {
3247 // TODO: Generalize this to work with other comparison idioms or ensure
3248 // they get canonicalized into this form.
3249
3250 // select i1 (a == b),
3251 // i32 Equal,
3252 // i32 (select i1 (a < b), i32 Less, i32 Greater)
3253 // where Equal, Less and Greater are placeholders for any three constants.
3254 CmpPredicate PredA;
3255 if (!match(SI->getCondition(), m_ICmp(PredA, m_Value(LHS), m_Value(RHS))) ||
3256 !ICmpInst::isEquality(PredA))
3257 return false;
3258 Value *EqualVal = SI->getTrueValue();
3259 Value *UnequalVal = SI->getFalseValue();
3260 // We still can get non-canonical predicate here, so canonicalize.
3261 if (PredA == ICmpInst::ICMP_NE)
3262 std::swap(EqualVal, UnequalVal);
3263 if (!match(EqualVal, m_ConstantInt(Equal)))
3264 return false;
3265 CmpPredicate PredB;
3266 Value *LHS2, *RHS2;
3267 if (!match(UnequalVal, m_Select(m_ICmp(PredB, m_Value(LHS2), m_Value(RHS2)),
3268 m_ConstantInt(Less), m_ConstantInt(Greater))))
3269 return false;
3270 // We can get predicate mismatch here, so canonicalize if possible:
3271 // First, ensure that 'LHS' match.
3272 if (LHS2 != LHS) {
3273 // x sgt y <--> y slt x
3274 std::swap(LHS2, RHS2);
3275 PredB = ICmpInst::getSwappedPredicate(PredB);
3276 }
3277 if (LHS2 != LHS)
3278 return false;
3279 // We also need to canonicalize 'RHS'.
3280 if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(RHS2)) {
3281 // x sgt C-1 <--> x sge C <--> not(x slt C)
3282 auto FlippedStrictness =
3284 PredB, cast<Constant>(RHS2));
3285 if (!FlippedStrictness)
3286 return false;
3287 assert(FlippedStrictness->first == ICmpInst::ICMP_SGE &&
3288 "basic correctness failure");
3289 RHS2 = FlippedStrictness->second;
3290 // And kind-of perform the result swap.
3291 std::swap(Less, Greater);
3292 PredB = ICmpInst::ICMP_SLT;
3293 }
3294 return PredB == ICmpInst::ICMP_SLT && RHS == RHS2;
3295}
3296
3299 ConstantInt *C) {
3300
3301 assert(C && "Cmp RHS should be a constant int!");
3302 // If we're testing a constant value against the result of a three way
3303 // comparison, the result can be expressed directly in terms of the
3304 // original values being compared. Note: We could possibly be more
3305 // aggressive here and remove the hasOneUse test. The original select is
3306 // really likely to simplify or sink when we remove a test of the result.
3307 Value *OrigLHS, *OrigRHS;
3308 ConstantInt *C1LessThan, *C2Equal, *C3GreaterThan;
3309 if (Cmp.hasOneUse() &&
3310 matchThreeWayIntCompare(Select, OrigLHS, OrigRHS, C1LessThan, C2Equal,
3311 C3GreaterThan)) {
3312 assert(C1LessThan && C2Equal && C3GreaterThan);
3313
3314 bool TrueWhenLessThan = ICmpInst::compare(
3315 C1LessThan->getValue(), C->getValue(), Cmp.getPredicate());
3316 bool TrueWhenEqual = ICmpInst::compare(C2Equal->getValue(), C->getValue(),
3317 Cmp.getPredicate());
3318 bool TrueWhenGreaterThan = ICmpInst::compare(
3319 C3GreaterThan->getValue(), C->getValue(), Cmp.getPredicate());
3320
3321 // This generates the new instruction that will replace the original Cmp
3322 // Instruction. Instead of enumerating the various combinations when
3323 // TrueWhenLessThan, TrueWhenEqual and TrueWhenGreaterThan are true versus
3324 // false, we rely on chaining of ORs and future passes of InstCombine to
3325 // simplify the OR further (i.e. a s< b || a == b becomes a s<= b).
3326
3327 // When none of the three constants satisfy the predicate for the RHS (C),
3328 // the entire original Cmp can be simplified to a false.
3330 if (TrueWhenLessThan)
3332 OrigLHS, OrigRHS));
3333 if (TrueWhenEqual)
3335 OrigLHS, OrigRHS));
3336 if (TrueWhenGreaterThan)
3338 OrigLHS, OrigRHS));
3339
3340 return replaceInstUsesWith(Cmp, Cond);
3341 }
3342 return nullptr;
3343}
3344
3346 auto *Bitcast = dyn_cast<BitCastInst>(Cmp.getOperand(0));
3347 if (!Bitcast)
3348 return nullptr;
3349
3350 ICmpInst::Predicate Pred = Cmp.getPredicate();
3351 Value *Op1 = Cmp.getOperand(1);
3352 Value *BCSrcOp = Bitcast->getOperand(0);
3353 Type *SrcType = Bitcast->getSrcTy();
3354 Type *DstType = Bitcast->getType();
3355
3356 // Make sure the bitcast doesn't change between scalar and vector and
3357 // doesn't change the number of vector elements.
3358 if (SrcType->isVectorTy() == DstType->isVectorTy() &&
3359 SrcType->getScalarSizeInBits() == DstType->getScalarSizeInBits()) {
3360 // Zero-equality and sign-bit checks are preserved through sitofp + bitcast.
3361 Value *X;
3362 if (match(BCSrcOp, m_SIToFP(m_Value(X)))) {
3363 // icmp eq (bitcast (sitofp X)), 0 --> icmp eq X, 0
3364 // icmp ne (bitcast (sitofp X)), 0 --> icmp ne X, 0
3365 // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0
3366 // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0
3367 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT ||
3368 Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) &&
3369 match(Op1, m_Zero()))
3370 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
3371
3372 // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1
3373 if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_One()))
3374 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), 1));
3375
3376 // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1
3377 if (Pred == ICmpInst::ICMP_SGT && match(Op1, m_AllOnes()))
3378 return new ICmpInst(Pred, X,
3379 ConstantInt::getAllOnesValue(X->getType()));
3380 }
3381
3382 // Zero-equality checks are preserved through unsigned floating-point casts:
3383 // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0
3384 // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0
3385 if (match(BCSrcOp, m_UIToFP(m_Value(X))))
3386 if (Cmp.isEquality() && match(Op1, m_Zero()))
3387 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
3388
3389 const APInt *C;
3390 bool TrueIfSigned;
3391 if (match(Op1, m_APInt(C)) && Bitcast->hasOneUse()) {
3392 // If this is a sign-bit test of a bitcast of a casted FP value, eliminate
3393 // the FP extend/truncate because that cast does not change the sign-bit.
3394 // This is true for all standard IEEE-754 types and the X86 80-bit type.
3395 // The sign-bit is always the most significant bit in those types.
3396 if (isSignBitCheck(Pred, *C, TrueIfSigned) &&
3397 (match(BCSrcOp, m_FPExt(m_Value(X))) ||
3398 match(BCSrcOp, m_FPTrunc(m_Value(X))))) {
3399 // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0
3400 // (bitcast (fpext/fptrunc X)) to iX) > -1 --> (bitcast X to iY) > -1
3401 Type *XType = X->getType();
3402
3403 // We can't currently handle Power style floating point operations here.
3404 if (!(XType->isPPC_FP128Ty() || SrcType->isPPC_FP128Ty())) {
3405 Type *NewType = Builder.getIntNTy(XType->getScalarSizeInBits());
3406 if (auto *XVTy = dyn_cast<VectorType>(XType))
3407 NewType = VectorType::get(NewType, XVTy->getElementCount());
3408 Value *NewBitcast = Builder.CreateBitCast(X, NewType);
3409 if (TrueIfSigned)
3410 return new ICmpInst(ICmpInst::ICMP_SLT, NewBitcast,
3411 ConstantInt::getNullValue(NewType));
3412 else
3413 return new ICmpInst(ICmpInst::ICMP_SGT, NewBitcast,
3415 }
3416 }
3417
3418 // icmp eq/ne (bitcast X to int), special fp -> llvm.is.fpclass(X, class)
3419 Type *FPType = SrcType->getScalarType();
3420 if (!Cmp.getParent()->getParent()->hasFnAttribute(
3421 Attribute::NoImplicitFloat) &&
3422 Cmp.isEquality() && FPType->isIEEELikeFPTy()) {
3423 FPClassTest Mask = APFloat(FPType->getFltSemantics(), *C).classify();
3424 if (Mask & (fcInf | fcZero)) {
3425 if (Pred == ICmpInst::ICMP_NE)
3426 Mask = ~Mask;
3427 return replaceInstUsesWith(Cmp,
3428 Builder.createIsFPClass(BCSrcOp, Mask));
3429 }
3430 }
3431 }
3432 }
3433
3434 const APInt *C;
3435 if (!match(Cmp.getOperand(1), m_APInt(C)) || !DstType->isIntegerTy() ||
3436 !SrcType->isIntOrIntVectorTy())
3437 return nullptr;
3438
3439 // If this is checking if all elements of a vector compare are set or not,
3440 // invert the casted vector equality compare and test if all compare
3441 // elements are clear or not. Compare against zero is generally easier for
3442 // analysis and codegen.
3443 // icmp eq/ne (bitcast (not X) to iN), -1 --> icmp eq/ne (bitcast X to iN), 0
3444 // Example: are all elements equal? --> are zero elements not equal?
3445 // TODO: Try harder to reduce compare of 2 freely invertible operands?
3446 if (Cmp.isEquality() && C->isAllOnes() && Bitcast->hasOneUse()) {
3447 if (Value *NotBCSrcOp =
3448 getFreelyInverted(BCSrcOp, BCSrcOp->hasOneUse(), &Builder)) {
3449 Value *Cast = Builder.CreateBitCast(NotBCSrcOp, DstType);
3450 return new ICmpInst(Pred, Cast, ConstantInt::getNullValue(DstType));
3451 }
3452 }
3453
3454 // If this is checking if all elements of an extended vector are clear or not,
3455 // compare in a narrow type to eliminate the extend:
3456 // icmp eq/ne (bitcast (ext X) to iN), 0 --> icmp eq/ne (bitcast X to iM), 0
3457 Value *X;
3458 if (Cmp.isEquality() && C->isZero() && Bitcast->hasOneUse() &&
3459 match(BCSrcOp, m_ZExtOrSExt(m_Value(X)))) {
3460 if (auto *VecTy = dyn_cast<FixedVectorType>(X->getType())) {
3461 Type *NewType = Builder.getIntNTy(VecTy->getPrimitiveSizeInBits());
3462 Value *NewCast = Builder.CreateBitCast(X, NewType);
3463 return new ICmpInst(Pred, NewCast, ConstantInt::getNullValue(NewType));
3464 }
3465 }
3466
3467 // Folding: icmp <pred> iN X, C
3468 // where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN
3469 // and C is a splat of a K-bit pattern
3470 // and SC is a constant vector = <C', C', C', ..., C'>
3471 // Into:
3472 // %E = extractelement <M x iK> %vec, i32 C'
3473 // icmp <pred> iK %E, trunc(C)
3474 Value *Vec;
3475 ArrayRef<int> Mask;
3476 if (match(BCSrcOp, m_Shuffle(m_Value(Vec), m_Undef(), m_Mask(Mask)))) {
3477 // Check whether every element of Mask is the same constant
3478 if (all_equal(Mask)) {
3479 auto *VecTy = cast<VectorType>(SrcType);
3480 auto *EltTy = cast<IntegerType>(VecTy->getElementType());
3481 if (C->isSplat(EltTy->getBitWidth())) {
3482 // Fold the icmp based on the value of C
3483 // If C is M copies of an iK sized bit pattern,
3484 // then:
3485 // => %E = extractelement <N x iK> %vec, i32 Elem
3486 // icmp <pred> iK %SplatVal, <pattern>
3487 Value *Elem = Builder.getInt32(Mask[0]);
3488 Value *Extract = Builder.CreateExtractElement(Vec, Elem);
3489 Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth()));
3490 return new ICmpInst(Pred, Extract, NewC);
3491 }
3492 }
3493 }
3494 return nullptr;
3495}
3496
3497/// Try to fold integer comparisons with a constant operand: icmp Pred X, C
3498/// where X is some kind of instruction.
3500 const APInt *C;
3501
3502 if (match(Cmp.getOperand(1), m_APInt(C))) {
3503 if (auto *BO = dyn_cast<BinaryOperator>(Cmp.getOperand(0)))
3504 if (Instruction *I = foldICmpBinOpWithConstant(Cmp, BO, *C))
3505 return I;
3506
3507 if (auto *SI = dyn_cast<SelectInst>(Cmp.getOperand(0)))
3508 // For now, we only support constant integers while folding the
3509 // ICMP(SELECT)) pattern. We can extend this to support vector of integers
3510 // similar to the cases handled by binary ops above.
3511 if (auto *ConstRHS = dyn_cast<ConstantInt>(Cmp.getOperand(1)))
3512 if (Instruction *I = foldICmpSelectConstant(Cmp, SI, ConstRHS))
3513 return I;
3514
3515 if (auto *TI = dyn_cast<TruncInst>(Cmp.getOperand(0)))
3516 if (Instruction *I = foldICmpTruncConstant(Cmp, TI, *C))
3517 return I;
3518
3519 if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0)))
3521 return I;
3522
3523 // (extractval ([s/u]subo X, Y), 0) == 0 --> X == Y
3524 // (extractval ([s/u]subo X, Y), 0) != 0 --> X != Y
3525 // TODO: This checks one-use, but that is not strictly necessary.
3526 Value *Cmp0 = Cmp.getOperand(0);
3527 Value *X, *Y;
3528 if (C->isZero() && Cmp.isEquality() && Cmp0->hasOneUse() &&
3529 (match(Cmp0,
3530 m_ExtractValue<0>(m_Intrinsic<Intrinsic::ssub_with_overflow>(
3531 m_Value(X), m_Value(Y)))) ||
3532 match(Cmp0,
3533 m_ExtractValue<0>(m_Intrinsic<Intrinsic::usub_with_overflow>(
3534 m_Value(X), m_Value(Y))))))
3535 return new ICmpInst(Cmp.getPredicate(), X, Y);
3536 }
3537
3538 if (match(Cmp.getOperand(1), m_APIntAllowPoison(C)))
3540
3541 return nullptr;
3542}
3543
3544/// Fold an icmp equality instruction with binary operator LHS and constant RHS:
3545/// icmp eq/ne BO, C.
3547 ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) {
3548 // TODO: Some of these folds could work with arbitrary constants, but this
3549 // function is limited to scalar and vector splat constants.
3550 if (!Cmp.isEquality())
3551 return nullptr;
3552
3553 ICmpInst::Predicate Pred = Cmp.getPredicate();
3554 bool isICMP_NE = Pred == ICmpInst::ICMP_NE;
3555 Constant *RHS = cast<Constant>(Cmp.getOperand(1));
3556 Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1);
3557
3558 switch (BO->getOpcode()) {
3559 case Instruction::SRem:
3560 // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one.
3561 if (C.isZero() && BO->hasOneUse()) {
3562 const APInt *BOC;
3563 if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) {
3564 Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName());
3565 return new ICmpInst(Pred, NewRem,
3567 }
3568 }
3569 break;
3570 case Instruction::Add: {
3571 // (A + C2) == C --> A == (C - C2)
3572 // (A + C2) != C --> A != (C - C2)
3573 // TODO: Remove the one-use limitation? See discussion in D58633.
3574 if (Constant *C2 = dyn_cast<Constant>(BOp1)) {
3575 if (BO->hasOneUse())
3576 return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(RHS, C2));
3577 } else if (C.isZero()) {
3578 // Replace ((add A, B) != 0) with (A != -B) if A or B is
3579 // efficiently invertible, or if the add has just this one use.
3580 if (Value *NegVal = dyn_castNegVal(BOp1))
3581 return new ICmpInst(Pred, BOp0, NegVal);
3582 if (Value *NegVal = dyn_castNegVal(BOp0))
3583 return new ICmpInst(Pred, NegVal, BOp1);
3584 if (BO->hasOneUse()) {
3585 // (add nuw A, B) != 0 -> (or A, B) != 0
3586 if (match(BO, m_NUWAdd(m_Value(), m_Value()))) {
3587 Value *Or = Builder.CreateOr(BOp0, BOp1);
3588 return new ICmpInst(Pred, Or, Constant::getNullValue(BO->getType()));
3589 }
3590 Value *Neg = Builder.CreateNeg(BOp1);
3591 Neg->takeName(BO);
3592 return new ICmpInst(Pred, BOp0, Neg);
3593 }
3594 }
3595 break;
3596 }
3597 case Instruction::Xor:
3598 if (Constant *BOC = dyn_cast<Constant>(BOp1)) {
3599 // For the xor case, we can xor two constants together, eliminating
3600 // the explicit xor.
3601 return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC));
3602 } else if (C.isZero()) {
3603 // Replace ((xor A, B) != 0) with (A != B)
3604 return new ICmpInst(Pred, BOp0, BOp1);
3605 }
3606 break;
3607 case Instruction::Or: {
3608 const APInt *BOC;
3609 if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) {
3610 // Comparing if all bits outside of a constant mask are set?
3611 // Replace (X | C) == -1 with (X & ~C) == ~C.
3612 // This removes the -1 constant.
3613 Constant *NotBOC = ConstantExpr::getNot(cast<Constant>(BOp1));
3614 Value *And = Builder.CreateAnd(BOp0, NotBOC);
3615 return new ICmpInst(Pred, And, NotBOC);
3616 }
3617 // (icmp eq (or (select cond, 0, NonZero), Other), 0)
3618 // -> (and cond, (icmp eq Other, 0))
3619 // (icmp ne (or (select cond, NonZero, 0), Other), 0)
3620 // -> (or cond, (icmp ne Other, 0))
3621 Value *Cond, *TV, *FV, *Other, *Sel;
3622 if (C.isZero() &&
3623 match(BO,
3626 m_Value(FV))),
3627 m_Value(Other)))) &&
3628 Cond->getType() == Cmp.getType()) {
3629 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
3630 // Easy case is if eq/ne matches whether 0 is trueval/falseval.
3631 if (Pred == ICmpInst::ICMP_EQ
3632 ? (match(TV, m_Zero()) && isKnownNonZero(FV, Q))
3633 : (match(FV, m_Zero()) && isKnownNonZero(TV, Q))) {
3634 Value *Cmp = Builder.CreateICmp(
3635 Pred, Other, Constant::getNullValue(Other->getType()));
3637 Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or, Cmp,
3638 Cond);
3639 }
3640 // Harder case is if eq/ne matches whether 0 is falseval/trueval. In this
3641 // case we need to invert the select condition so we need to be careful to
3642 // avoid creating extra instructions.
3643 // (icmp ne (or (select cond, 0, NonZero), Other), 0)
3644 // -> (or (not cond), (icmp ne Other, 0))
3645 // (icmp eq (or (select cond, NonZero, 0), Other), 0)
3646 // -> (and (not cond), (icmp eq Other, 0))
3647 //
3648 // Only do this if the inner select has one use, in which case we are
3649 // replacing `select` with `(not cond)`. Otherwise, we will create more
3650 // uses. NB: Trying to freely invert cond doesn't make sense here, as if
3651 // cond was freely invertable, the select arms would have been inverted.
3652 if (Sel->hasOneUse() &&
3653 (Pred == ICmpInst::ICMP_EQ
3654 ? (match(FV, m_Zero()) && isKnownNonZero(TV, Q))
3655 : (match(TV, m_Zero()) && isKnownNonZero(FV, Q)))) {
3656 Value *NotCond = Builder.CreateNot(Cond);
3657 Value *Cmp = Builder.CreateICmp(
3658 Pred, Other, Constant::getNullValue(Other->getType()));
3660 Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or, Cmp,
3661 NotCond);
3662 }
3663 }
3664 break;
3665 }
3666 case Instruction::UDiv:
3667 case Instruction::SDiv:
3668 if (BO->isExact()) {
3669 // div exact X, Y eq/ne 0 -> X eq/ne 0
3670 // div exact X, Y eq/ne 1 -> X eq/ne Y
3671 // div exact X, Y eq/ne C ->
3672 // if Y * C never-overflow && OneUse:
3673 // -> Y * C eq/ne X
3674 if (C.isZero())
3675 return new ICmpInst(Pred, BOp0, Constant::getNullValue(BO->getType()));
3676 else if (C.isOne())
3677 return new ICmpInst(Pred, BOp0, BOp1);
3678 else if (BO->hasOneUse()) {
3680 Instruction::Mul, BO->getOpcode() == Instruction::SDiv, BOp1,
3681 Cmp.getOperand(1), BO);
3683 Value *YC =
3684 Builder.CreateMul(BOp1, ConstantInt::get(BO->getType(), C));
3685 return new ICmpInst(Pred, YC, BOp0);
3686 }
3687 }
3688 }
3689 if (BO->getOpcode() == Instruction::UDiv && C.isZero()) {
3690 // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A)
3691 auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT;
3692 return new ICmpInst(NewPred, BOp1, BOp0);
3693 }
3694 break;
3695 default:
3696 break;
3697 }
3698 return nullptr;
3699}
3700
3702 const APInt &CRhs,
3703 InstCombiner::BuilderTy &Builder,
3704 const SimplifyQuery &Q) {
3705 assert(CtpopLhs->getIntrinsicID() == Intrinsic::ctpop &&
3706 "Non-ctpop intrin in ctpop fold");
3707 if (!CtpopLhs->hasOneUse())
3708 return nullptr;
3709
3710 // Power of 2 test:
3711 // isPow2OrZero : ctpop(X) u< 2
3712 // isPow2 : ctpop(X) == 1
3713 // NotPow2OrZero: ctpop(X) u> 1
3714 // NotPow2 : ctpop(X) != 1
3715 // If we know any bit of X can be folded to:
3716 // IsPow2 : X & (~Bit) == 0
3717 // NotPow2 : X & (~Bit) != 0
3718 const ICmpInst::Predicate Pred = I.getPredicate();
3719 if (((I.isEquality() || Pred == ICmpInst::ICMP_UGT) && CRhs == 1) ||
3720 (Pred == ICmpInst::ICMP_ULT && CRhs == 2)) {
3721 Value *Op = CtpopLhs->getArgOperand(0);
3722 KnownBits OpKnown = computeKnownBits(Op, Q.DL,
3723 /*Depth*/ 0, Q.AC, Q.CxtI, Q.DT);
3724 // No need to check for count > 1, that should be already constant folded.
3725 if (OpKnown.countMinPopulation() == 1) {
3726 Value *And = Builder.CreateAnd(
3727 Op, Constant::getIntegerValue(Op->getType(), ~(OpKnown.One)));
3728 return new ICmpInst(
3729 (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_ULT)
3732 And, Constant::getNullValue(Op->getType()));
3733 }
3734 }
3735
3736 return nullptr;
3737}
3738
3739/// Fold an equality icmp with LLVM intrinsic and constant operand.
3741 ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) {
3742 Type *Ty = II->getType();
3743 unsigned BitWidth = C.getBitWidth();
3744 const ICmpInst::Predicate Pred = Cmp.getPredicate();
3745
3746 switch (II->getIntrinsicID()) {
3747 case Intrinsic::abs:
3748 // abs(A) == 0 -> A == 0
3749 // abs(A) == INT_MIN -> A == INT_MIN
3750 if (C.isZero() || C.isMinSignedValue())
3751 return new ICmpInst(Pred, II->getArgOperand(0), ConstantInt::get(Ty, C));
3752 break;
3753
3754 case Intrinsic::bswap:
3755 // bswap(A) == C -> A == bswap(C)
3756 return new ICmpInst(Pred, II->getArgOperand(0),
3757 ConstantInt::get(Ty, C.byteSwap()));
3758
3759 case Intrinsic::bitreverse:
3760 // bitreverse(A) == C -> A == bitreverse(C)
3761 return new ICmpInst(Pred, II->getArgOperand(0),
3762 ConstantInt::get(Ty, C.reverseBits()));
3763
3764 case Intrinsic::ctlz:
3765 case Intrinsic::cttz: {
3766 // ctz(A) == bitwidth(A) -> A == 0 and likewise for !=
3767 if (C == BitWidth)
3768 return new ICmpInst(Pred, II->getArgOperand(0),
3770
3771 // ctz(A) == C -> A & Mask1 == Mask2, where Mask2 only has bit C set
3772 // and Mask1 has bits 0..C+1 set. Similar for ctl, but for high bits.
3773 // Limit to one use to ensure we don't increase instruction count.
3774 unsigned Num = C.getLimitedValue(BitWidth);
3775 if (Num != BitWidth && II->hasOneUse()) {
3776 bool IsTrailing = II->getIntrinsicID() == Intrinsic::cttz;
3777 APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(BitWidth, Num + 1)
3778 : APInt::getHighBitsSet(BitWidth, Num + 1);
3779 APInt Mask2 = IsTrailing
3782 return new ICmpInst(Pred, Builder.CreateAnd(II->getArgOperand(0), Mask1),
3783 ConstantInt::get(Ty, Mask2));
3784 }
3785 break;
3786 }
3787
3788 case Intrinsic::ctpop: {
3789 // popcount(A) == 0 -> A == 0 and likewise for !=
3790 // popcount(A) == bitwidth(A) -> A == -1 and likewise for !=
3791 bool IsZero = C.isZero();
3792 if (IsZero || C == BitWidth)
3793 return new ICmpInst(Pred, II->getArgOperand(0),
3794 IsZero ? Constant::getNullValue(Ty)
3796
3797 break;
3798 }
3799
3800 case Intrinsic::fshl:
3801 case Intrinsic::fshr:
3802 if (II->getArgOperand(0) == II->getArgOperand(1)) {
3803 const APInt *RotAmtC;
3804 // ror(X, RotAmtC) == C --> X == rol(C, RotAmtC)
3805 // rol(X, RotAmtC) == C --> X == ror(C, RotAmtC)
3806 if (match(II->getArgOperand(2), m_APInt(RotAmtC)))
3807 return new ICmpInst(Pred, II->getArgOperand(0),
3808 II->getIntrinsicID() == Intrinsic::fshl
3809 ? ConstantInt::get(Ty, C.rotr(*RotAmtC))
3810 : ConstantInt::get(Ty, C.rotl(*RotAmtC)));
3811 }
3812 break;
3813
3814 case Intrinsic::umax:
3815 case Intrinsic::uadd_sat: {
3816 // uadd.sat(a, b) == 0 -> (a | b) == 0
3817 // umax(a, b) == 0 -> (a | b) == 0
3818 if (C.isZero() && II->hasOneUse()) {
3819 Value *Or = Builder.CreateOr(II->getArgOperand(0), II->getArgOperand(1));
3820 return new ICmpInst(Pred, Or, Constant::getNullValue(Ty));
3821 }
3822 break;
3823 }
3824
3825 case Intrinsic::ssub_sat:
3826 // ssub.sat(a, b) == 0 -> a == b
3827 if (C.isZero())
3828 return new ICmpInst(Pred, II->getArgOperand(0), II->getArgOperand(1));
3829 break;
3830 case Intrinsic::usub_sat: {
3831 // usub.sat(a, b) == 0 -> a <= b
3832 if (C.isZero()) {
3833 ICmpInst::Predicate NewPred =
3835 return new ICmpInst(NewPred, II->getArgOperand(0), II->getArgOperand(1));
3836 }
3837 break;
3838 }
3839 default:
3840 break;
3841 }
3842
3843 return nullptr;
3844}
3845
3846/// Fold an icmp with LLVM intrinsics
3847static Instruction *
3849 InstCombiner::BuilderTy &Builder) {
3850 assert(Cmp.isEquality());
3851
3852 ICmpInst::Predicate Pred = Cmp.getPredicate();
3853 Value *Op0 = Cmp.getOperand(0);
3854 Value *Op1 = Cmp.getOperand(1);
3855 const auto *IIOp0 = dyn_cast<IntrinsicInst>(Op0);
3856 const auto *IIOp1 = dyn_cast<IntrinsicInst>(Op1);
3857 if (!IIOp0 || !IIOp1 || IIOp0->getIntrinsicID() != IIOp1->getIntrinsicID())
3858 return nullptr;
3859
3860 switch (IIOp0->getIntrinsicID()) {
3861 case Intrinsic::bswap:
3862 case Intrinsic::bitreverse:
3863 // If both operands are byte-swapped or bit-reversed, just compare the
3864 // original values.
3865 return new ICmpInst(Pred, IIOp0->getOperand(0), IIOp1->getOperand(0));
3866 case Intrinsic::fshl:
3867 case Intrinsic::fshr: {
3868 // If both operands are rotated by same amount, just compare the
3869 // original values.
3870 if (IIOp0->getOperand(0) != IIOp0->getOperand(1))
3871 break;
3872 if (IIOp1->getOperand(0) != IIOp1->getOperand(1))
3873 break;
3874 if (IIOp0->getOperand(2) == IIOp1->getOperand(2))
3875 return new ICmpInst(Pred, IIOp0->getOperand(0), IIOp1->getOperand(0));
3876
3877 // rotate(X, AmtX) == rotate(Y, AmtY)
3878 // -> rotate(X, AmtX - AmtY) == Y
3879 // Do this if either both rotates have one use or if only one has one use
3880 // and AmtX/AmtY are constants.
3881 unsigned OneUses = IIOp0->hasOneUse() + IIOp1->hasOneUse();
3882 if (OneUses == 2 ||
3883 (OneUses == 1 && match(IIOp0->getOperand(2), m_ImmConstant()) &&
3884 match(IIOp1->getOperand(2), m_ImmConstant()))) {
3885 Value *SubAmt =
3886 Builder.CreateSub(IIOp0->getOperand(2), IIOp1->getOperand(2));
3887 Value *CombinedRotate = Builder.CreateIntrinsic(
3888 Op0->getType(), IIOp0->getIntrinsicID(),
3889 {IIOp0->getOperand(0), IIOp0->getOperand(0), SubAmt});
3890 return new ICmpInst(Pred, IIOp1->getOperand(0), CombinedRotate);
3891 }
3892 } break;
3893 default:
3894 break;
3895 }
3896
3897 return nullptr;
3898}
3899
3900/// Try to fold integer comparisons with a constant operand: icmp Pred X, C
3901/// where X is some kind of instruction and C is AllowPoison.
3902/// TODO: Move more folds which allow poison to this function.
3905 const APInt &C) {
3906 const ICmpInst::Predicate Pred = Cmp.getPredicate();
3907 if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0))) {
3908 switch (II->getIntrinsicID()) {
3909 default:
3910 break;
3911 case Intrinsic::fshl:
3912 case Intrinsic::fshr:
3913 if (Cmp.isEquality() && II->getArgOperand(0) == II->getArgOperand(1)) {
3914 // (rot X, ?) == 0/-1 --> X == 0/-1
3915 if (C.isZero() || C.isAllOnes())
3916 return new ICmpInst(Pred, II->getArgOperand(0), Cmp.getOperand(1));
3917 }
3918 break;
3919 }
3920 }
3921
3922 return nullptr;
3923}
3924
3925/// Fold an icmp with BinaryOp and constant operand: icmp Pred BO, C.
3927 BinaryOperator *BO,
3928 const APInt &C) {
3929 switch (BO->getOpcode()) {
3930 case Instruction::Xor:
3931 if (Instruction *I = foldICmpXorConstant(Cmp, BO, C))
3932 return I;
3933 break;
3934 case Instruction::And:
3935 if (Instruction *I = foldICmpAndConstant(Cmp, BO, C))
3936 return I;
3937 break;
3938 case Instruction::Or:
3939 if (Instruction *I = foldICmpOrConstant(Cmp, BO, C))
3940 return I;
3941 break;
3942 case Instruction::Mul:
3943 if (Instruction *I = foldICmpMulConstant(Cmp, BO, C))
3944 return I;
3945 break;
3946 case Instruction::Shl:
3947 if (Instruction *I = foldICmpShlConstant(Cmp, BO, C))
3948 return I;
3949 break;
3950 case Instruction::LShr:
3951 case Instruction::AShr:
3952 if (Instruction *I = foldICmpShrConstant(Cmp, BO, C))
3953 return I;
3954 break;
3955 case Instruction::SRem:
3956 if (Instruction *I = foldICmpSRemConstant(Cmp, BO, C))
3957 return I;
3958 break;
3959 case Instruction::UDiv:
3960 if (Instruction *I = foldICmpUDivConstant(Cmp, BO, C))
3961 return I;
3962 [[fallthrough]];
3963 case Instruction::SDiv:
3964 if (Instruction *I = foldICmpDivConstant(Cmp, BO, C))
3965 return I;
3966 break;
3967 case Instruction::Sub:
3968 if (Instruction *I = foldICmpSubConstant(Cmp, BO, C))
3969 return I;
3970 break;
3971 case Instruction::Add:
3972 if (Instruction *I = foldICmpAddConstant(Cmp, BO, C))
3973 return I;
3974 break;
3975 default:
3976 break;
3977 }
3978
3979 // TODO: These folds could be refactored to be part of the above calls.
3980 return foldICmpBinOpEqualityWithConstant(Cmp, BO, C);
3981}
3982
3983static Instruction *
3985 const APInt &C,
3986 InstCombiner::BuilderTy &Builder) {
3987 // This transform may end up producing more than one instruction for the
3988 // intrinsic, so limit it to one user of the intrinsic.
3989 if (!II->hasOneUse())
3990 return nullptr;
3991
3992 // Let Y = [add/sub]_sat(X, C) pred C2
3993 // SatVal = The saturating value for the operation
3994 // WillWrap = Whether or not the operation will underflow / overflow
3995 // => Y = (WillWrap ? SatVal : (X binop C)) pred C2
3996 // => Y = WillWrap ? (SatVal pred C2) : ((X binop C) pred C2)
3997 //
3998 // When (SatVal pred C2) is true, then
3999 // Y = WillWrap ? true : ((X binop C) pred C2)
4000 // => Y = WillWrap || ((X binop C) pred C2)
4001 // else
4002 // Y = WillWrap ? false : ((X binop C) pred C2)
4003 // => Y = !WillWrap ? ((X binop C) pred C2) : false
4004 // => Y = !WillWrap && ((X binop C) pred C2)
4005 Value *Op0 = II->getOperand(0);
4006 Value *Op1 = II->getOperand(1);
4007
4008 const APInt *COp1;
4009 // This transform only works when the intrinsic has an integral constant or
4010 // splat vector as the second operand.
4011 if (!match(Op1, m_APInt(COp1)))
4012 return nullptr;
4013
4014 APInt SatVal;
4015 switch (II->getIntrinsicID()) {
4016 default:
4018 "This function only works with usub_sat and uadd_sat for now!");
4019 case Intrinsic::uadd_sat:
4020 SatVal = APInt::getAllOnes(C.getBitWidth());
4021 break;
4022 case Intrinsic::usub_sat:
4023 SatVal = APInt::getZero(C.getBitWidth());
4024 break;
4025 }
4026
4027 // Check (SatVal pred C2)
4028 bool SatValCheck = ICmpInst::compare(SatVal, C, Pred);
4029
4030 // !WillWrap.
4032 II->getBinaryOp(), *COp1, II->getNoWrapKind());
4033
4034 // WillWrap.
4035 if (SatValCheck)
4036 C1 = C1.inverse();
4037
4039 if (II->getBinaryOp() == Instruction::Add)
4040 C2 = C2.sub(*COp1);
4041 else
4042 C2 = C2.add(*COp1);
4043
4044 Instruction::BinaryOps CombiningOp =
4045 SatValCheck ? Instruction::BinaryOps::Or : Instruction::BinaryOps::And;
4046
4047 std::optional<ConstantRange> Combination;
4048 if (CombiningOp == Instruction::BinaryOps::Or)
4049 Combination = C1.exactUnionWith(C2);
4050 else /* CombiningOp == Instruction::BinaryOps::And */
4051 Combination = C1.exactIntersectWith(C2);
4052
4053 if (!Combination)
4054 return nullptr;
4055
4056 CmpInst::Predicate EquivPred;
4057 APInt EquivInt;
4058 APInt EquivOffset;
4059
4060 Combination->getEquivalentICmp(EquivPred, EquivInt, EquivOffset);
4061
4062 return new ICmpInst(
4063 EquivPred,
4064 Builder.CreateAdd(Op0, ConstantInt::get(Op1->getType(), EquivOffset)),
4065 ConstantInt::get(Op1->getType(), EquivInt));
4066}
4067
4068static Instruction *
4070 const APInt &C,
4071 InstCombiner::BuilderTy &Builder) {
4072 std::optional<ICmpInst::Predicate> NewPredicate = std::nullopt;
4073 switch (Pred) {
4074 case ICmpInst::ICMP_EQ:
4075 case ICmpInst::ICMP_NE:
4076 if (C.isZero())
4077 NewPredicate = Pred;
4078 else if (C.isOne())
4079 NewPredicate =
4081 else if (C.isAllOnes())
4082 NewPredicate =
4084 break;
4085
4086 case ICmpInst::ICMP_SGT:
4087 if (C.isAllOnes())
4088 NewPredicate = ICmpInst::ICMP_UGE;
4089 else if (C.isZero())
4090 NewPredicate = ICmpInst::ICMP_UGT;
4091 break;
4092
4093 case ICmpInst::ICMP_SLT:
4094 if (C.isZero())
4095 NewPredicate = ICmpInst::ICMP_ULT;
4096 else if (C.isOne())
4097 NewPredicate = ICmpInst::ICMP_ULE;
4098 break;
4099
4100 case ICmpInst::ICMP_ULT:
4101 if (C.ugt(1))
4102 NewPredicate = ICmpInst::ICMP_UGE;
4103 break;
4104
4105 case ICmpInst::ICMP_UGT:
4106 if (!C.isZero() && !C.isAllOnes())
4107 NewPredicate = ICmpInst::ICMP_ULT;
4108 break;
4109
4110 default:
4111 break;
4112 }
4113
4114 if (!NewPredicate)
4115 return nullptr;
4116
4117 if (I->getIntrinsicID() == Intrinsic::scmp)
4118 NewPredicate = ICmpInst::getSignedPredicate(*NewPredicate);
4119 Value *LHS = I->getOperand(0);
4120 Value *RHS = I->getOperand(1);
4121 return new ICmpInst(*NewPredicate, LHS, RHS);
4122}
4123
4124/// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
4127 const APInt &C) {
4128 ICmpInst::Predicate Pred = Cmp.getPredicate();
4129
4130 // Handle folds that apply for any kind of icmp.
4131 switch (II->getIntrinsicID()) {
4132 default:
4133 break;
4134 case Intrinsic::uadd_sat:
4135 case Intrinsic::usub_sat:
4136 if (auto *Folded = foldICmpUSubSatOrUAddSatWithConstant(
4137 Pred, cast<SaturatingInst>(II), C, Builder))
4138 return Folded;
4139 break;
4140 case Intrinsic::ctpop: {
4141 const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
4142 if (Instruction *R = foldCtpopPow2Test(Cmp, II, C, Builder, Q))
4143 return R;
4144 } break;
4145 case Intrinsic::scmp:
4146 case Intrinsic::ucmp:
4147 if (auto *Folded = foldICmpOfCmpIntrinsicWithConstant(Pred, II, C, Builder))
4148 return Folded;
4149 break;
4150 }
4151
4152 if (Cmp.isEquality())
4153 return foldICmpEqIntrinsicWithConstant(Cmp, II, C);
4154
4155 Type *Ty = II->getType();
4156 unsigned BitWidth = C.getBitWidth();
4157 switch (II->getIntrinsicID()) {
4158 case Intrinsic::ctpop: {
4159 // (ctpop X > BitWidth - 1) --> X == -1
4160 Value *X = II->getArgOperand(0);
4161 if (C == BitWidth - 1 && Pred == ICmpInst::ICMP_UGT)
4162 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, X,
4164 // (ctpop X < BitWidth) --> X != -1
4165 if (C == BitWidth && Pred == ICmpInst::ICMP_ULT)
4166 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, X,
4168 break;
4169 }
4170 case Intrinsic::ctlz: {
4171 // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000
4172 if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) {
4173 unsigned Num = C.getLimitedValue();
4174 APInt Limit = APInt::getOneBitSet(BitWidth, BitWidth - Num - 1);
4175 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT,
4176 II->getArgOperand(0), ConstantInt::get(Ty, Limit));
4177 }
4178
4179 // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111
4180 if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) {
4181 unsigned Num = C.getLimitedValue();
4183 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT,
4184 II->getArgOperand(0), ConstantInt::get(Ty, Limit));
4185 }
4186 break;
4187 }
4188 case Intrinsic::cttz: {
4189 // Limit to one use to ensure we don't increase instruction count.
4190 if (!II->hasOneUse())
4191 return nullptr;
4192
4193 // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0
4194 if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) {
4195 APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue() + 1);
4196 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ,
4197 Builder.CreateAnd(II->getArgOperand(0), Mask),
4199 }
4200
4201 // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0
4202 if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) {
4203 APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue());
4204 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE,
4205 Builder.CreateAnd(II->getArgOperand(0), Mask),
4207 }
4208 break;
4209 }
4210 case Intrinsic::ssub_sat:
4211 // ssub.sat(a, b) spred 0 -> a spred b
4212 if (ICmpInst::isSigned(Pred)) {
4213 if (C.isZero())
4214 return new ICmpInst(Pred, II->getArgOperand(0), II->getArgOperand(1));
4215 // X s<= 0 is cannonicalized to X s< 1
4216 if (Pred == ICmpInst::ICMP_SLT && C.isOne())
4217 return new ICmpInst(ICmpInst::ICMP_SLE, II->getArgOperand(0),
4218 II->getArgOperand(1));
4219 // X s>= 0 is cannonicalized to X s> -1
4220 if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
4221 return new ICmpInst(ICmpInst::ICMP_SGE, II->getArgOperand(0),
4222 II->getArgOperand(1));
4223 }
4224 break;
4225 default:
4226 break;
4227 }
4228
4229 return nullptr;
4230}
4231
4232/// Handle icmp with constant (but not simple integer constant) RHS.
4234 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
4235 Constant *RHSC = dyn_cast<Constant>(Op1);
4236 Instruction *LHSI = dyn_cast<Instruction>(Op0);
4237 if (!RHSC || !LHSI)
4238 return nullptr;
4239
4240 switch (LHSI->getOpcode()) {
4241 case Instruction::PHI:
4242 if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI)))
4243 return NV;
4244 break;
4245 case Instruction::IntToPtr:
4246 // icmp pred inttoptr(X), null -> icmp pred X, 0
4247 if (RHSC->isNullValue() &&
4248 DL.getIntPtrType(RHSC->getType()) == LHSI->getOperand(0)->getType())
4249 return new ICmpInst(
4250 I.getPredicate(), LHSI->getOperand(0),
4252 break;
4253
4254 case Instruction::Load:
4255 // Try to optimize things like "A[i] > 4" to index computations.
4256 if (GetElementPtrInst *GEP =
4257 dyn_cast<GetElementPtrInst>(LHSI->getOperand(0)))
4258 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)))
4259 if (Instruction *Res =
4260 foldCmpLoadFromIndexedGlobal(cast<LoadInst>(LHSI), GEP, GV, I))
4261 return Res;
4262 break;
4263 }
4264
4265 return nullptr;
4266}
4267
4269 Value *RHS, const ICmpInst &I) {
4270 // Try to fold the comparison into the select arms, which will cause the
4271 // select to be converted into a logical and/or.
4272 auto SimplifyOp = [&](Value *Op, bool SelectCondIsTrue) -> Value * {
4273 if (Value *Res = simplifyICmpInst(Pred, Op, RHS, SQ))
4274 return Res;
4275 if (std::optional<bool> Impl = isImpliedCondition(
4276 SI->getCondition(), Pred, Op, RHS, DL, SelectCondIsTrue))
4277 return ConstantInt::get(I.getType(), *Impl);
4278 return nullptr;
4279 };
4280
4281 ConstantInt *CI = nullptr;
4282 Value *Op1 = SimplifyOp(SI->getOperand(1), true);
4283 if (Op1)
4284 CI = dyn_cast<ConstantInt>(Op1);
4285
4286 Value *Op2 = SimplifyOp(SI->getOperand(2), false);
4287 if (Op2)
4288 CI = dyn_cast<ConstantInt>(Op2);
4289
4290 auto Simplifies = [&](Value *Op, unsigned Idx) {
4291 // A comparison of ucmp/scmp with a constant will fold into an icmp.
4292 const APInt *Dummy;
4293 return Op ||
4294 (isa<CmpIntrinsic>(SI->getOperand(Idx)) &&
4295 SI->getOperand(Idx)->hasOneUse() && match(RHS, m_APInt(Dummy)));
4296 };
4297
4298 // We only want to perform this transformation if it will not lead to
4299 // additional code. This is true if either both sides of the select
4300 // fold to a constant (in which case the icmp is replaced with a select
4301 // which will usually simplify) or this is the only user of the
4302 // select (in which case we are trading a select+icmp for a simpler
4303 // select+icmp) or all uses of the select can be replaced based on
4304 // dominance information ("Global cases").
4305 bool Transform = false;
4306 if (Op1 && Op2)
4307 Transform = true;
4308 else if (Simplifies(Op1, 1) || Simplifies(Op2, 2)) {
4309 // Local case
4310 if (SI->hasOneUse())
4311 Transform = true;
4312 // Global cases
4313 else if (CI && !CI->isZero())
4314 // When Op1 is constant try replacing select with second operand.
4315 // Otherwise Op2 is constant and try replacing select with first
4316 // operand.
4317 Transform = replacedSelectWithOperand(SI, &I, Op1 ? 2 : 1);
4318 }
4319 if (Transform) {
4320 if (!Op1)
4321 Op1 = Builder.CreateICmp(Pred, SI->getOperand(1), RHS, I.getName());
4322 if (!Op2)
4323 Op2 = Builder.CreateICmp(Pred, SI->getOperand(2), RHS, I.getName());
4324 return SelectInst::Create(SI->getOperand(0), Op1, Op2);
4325 }
4326
4327 return nullptr;
4328}
4329
4330// Returns whether V is a Mask ((X + 1) & X == 0) or ~Mask (-Pow2OrZero)
4331static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q,
4332 unsigned Depth = 0) {
4333 if (Not ? match(V, m_NegatedPower2OrZero()) : match(V, m_LowBitMaskOrZero()))
4334 return true;
4335 if (V->getType()->getScalarSizeInBits() == 1)
4336 return true;
4338 return false;
4339 Value *X;
4340 const Instruction *I = dyn_cast<Instruction>(V);
4341 if (!I)
4342 return false;
4343 switch (I->getOpcode()) {
4344 case Instruction::ZExt:
4345 // ZExt(Mask) is a Mask.
4346 return !Not && isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4347 case Instruction::SExt:
4348 // SExt(Mask) is a Mask.
4349 // SExt(~Mask) is a ~Mask.
4350 return isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4351 case Instruction::And:
4352 case Instruction::Or:
4353 // Mask0 | Mask1 is a Mask.
4354 // Mask0 & Mask1 is a Mask.
4355 // ~Mask0 | ~Mask1 is a ~Mask.
4356 // ~Mask0 & ~Mask1 is a ~Mask.
4357 return isMaskOrZero(I->getOperand(1), Not, Q, Depth) &&
4358 isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4359 case Instruction::Xor:
4360 if (match(V, m_Not(m_Value(X))))
4361 return isMaskOrZero(X, !Not, Q, Depth);
4362
4363 // (X ^ -X) is a ~Mask
4364 if (Not)
4365 return match(V, m_c_Xor(m_Value(X), m_Neg(m_Deferred(X))));
4366 // (X ^ (X - 1)) is a Mask
4367 else
4368 return match(V, m_c_Xor(m_Value(X), m_Add(m_Deferred(X), m_AllOnes())));
4369 case Instruction::Select:
4370 // c ? Mask0 : Mask1 is a Mask.
4371 return isMaskOrZero(I->getOperand(1), Not, Q, Depth) &&
4372 isMaskOrZero(I->getOperand(2), Not, Q, Depth);
4373 case Instruction::Shl:
4374 // (~Mask) << X is a ~Mask.
4375 return Not && isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4376 case Instruction::LShr:
4377 // Mask >> X is a Mask.
4378 return !Not && isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4379 case Instruction::AShr:
4380 // Mask s>> X is a Mask.
4381 // ~Mask s>> X is a ~Mask.
4382 return isMaskOrZero(I->getOperand(0), Not, Q, Depth);
4383 case Instruction::Add:
4384 // Pow2 - 1 is a Mask.
4385 if (!Not && match(I->getOperand(1), m_AllOnes()))
4386 return isKnownToBeAPowerOfTwo(I->getOperand(0), Q.DL, /*OrZero*/ true,
4387 Depth, Q.AC, Q.CxtI, Q.DT);
4388 break;
4389 case Instruction::Sub:
4390 // -Pow2 is a ~Mask.
4391 if (Not && match(I->getOperand(0), m_Zero()))
4392 return isKnownToBeAPowerOfTwo(I->getOperand(1), Q.DL, /*OrZero*/ true,
4393 Depth, Q.AC, Q.CxtI, Q.DT);
4394 break;
4395 case Instruction::Call: {
4396 if (auto *II = dyn_cast<IntrinsicInst>(I)) {
4397 switch (II->getIntrinsicID()) {
4398 // min/max(Mask0, Mask1) is a Mask.
4399 // min/max(~Mask0, ~Mask1) is a ~Mask.
4400 case Intrinsic::umax:
4401 case Intrinsic::smax:
4402 case Intrinsic::umin:
4403 case Intrinsic::smin:
4404 return isMaskOrZero(II->getArgOperand(1), Not, Q, Depth) &&
4405 isMaskOrZero(II->getArgOperand(0), Not, Q, Depth);
4406
4407 // In the context of masks, bitreverse(Mask) == ~Mask
4408 case Intrinsic::bitreverse:
4409 return isMaskOrZero(II->getArgOperand(0), !Not, Q, Depth);
4410 default:
4411 break;
4412 }
4413 }
4414 break;
4415 }
4416 default:
4417 break;
4418 }
4419 return false;
4420}
4421
4422/// Some comparisons can be simplified.
4423/// In this case, we are looking for comparisons that look like
4424/// a check for a lossy truncation.
4425/// Folds:
4426/// icmp SrcPred (x & Mask), x to icmp DstPred x, Mask
4427/// icmp SrcPred (x & ~Mask), ~Mask to icmp DstPred x, ~Mask
4428/// icmp eq/ne (x & ~Mask), 0 to icmp DstPred x, Mask
4429/// icmp eq/ne (~x | Mask), -1 to icmp DstPred x, Mask
4430/// Where Mask is some pattern that produces all-ones in low bits:
4431/// (-1 >> y)
4432/// ((-1 << y) >> y) <- non-canonical, has extra uses
4433/// ~(-1 << y)
4434/// ((1 << y) + (-1)) <- non-canonical, has extra uses
4435/// The Mask can be a constant, too.
4436/// For some predicates, the operands are commutative.
4437/// For others, x can only be on a specific side.
4439 Value *Op1, const SimplifyQuery &Q,
4440 InstCombiner &IC) {
4441
4442 ICmpInst::Predicate DstPred;
4443 switch (Pred) {
4445 // x & Mask == x
4446 // x & ~Mask == 0
4447 // ~x | Mask == -1
4448 // -> x u<= Mask
4449 // x & ~Mask == ~Mask
4450 // -> ~Mask u<= x
4452 break;
4454 // x & Mask != x
4455 // x & ~Mask != 0
4456 // ~x | Mask != -1
4457 // -> x u> Mask
4458 // x & ~Mask != ~Mask
4459 // -> ~Mask u> x
4461 break;
4463 // x & Mask u< x
4464 // -> x u> Mask
4465 // x & ~Mask u< ~Mask
4466 // -> ~Mask u> x
4468 break;
4470 // x & Mask u>= x
4471 // -> x u<= Mask
4472 // x & ~Mask u>= ~Mask
4473 // -> ~Mask u<= x
4475 break;
4477 // x & Mask s< x [iff Mask s>= 0]
4478 // -> x s> Mask
4479 // x & ~Mask s< ~Mask [iff ~Mask != 0]
4480 // -> ~Mask s> x
4482 break;
4484 // x & Mask s>= x [iff Mask s>= 0]
4485 // -> x s<= Mask
4486 // x & ~Mask s>= ~Mask [iff ~Mask != 0]
4487 // -> ~Mask s<= x
4489 break;
4490 default:
4491 // We don't support sgt,sle
4492 // ult/ugt are simplified to true/false respectively.
4493 return nullptr;
4494 }
4495
4496 Value *X, *M;
4497 // Put search code in lambda for early positive returns.
4498 auto IsLowBitMask = [&]() {
4499 if (match(Op0, m_c_And(m_Specific(Op1), m_Value(M)))) {
4500 X = Op1;
4501 // Look for: x & Mask pred x
4502 if (isMaskOrZero(M, /*Not=*/false, Q)) {
4503 return !ICmpInst::isSigned(Pred) ||
4504 (match(M, m_NonNegative()) || isKnownNonNegative(M, Q));
4505 }
4506
4507 // Look for: x & ~Mask pred ~Mask
4508 if (isMaskOrZero(X, /*Not=*/true, Q)) {
4509 return !ICmpInst::isSigned(Pred) || isKnownNonZero(X, Q);
4510 }
4511 return false;
4512 }
4513 if (ICmpInst::isEquality(Pred) && match(Op1, m_AllOnes()) &&
4514 match(Op0, m_OneUse(m_Or(m_Value(X), m_Value(M))))) {
4515
4516 auto Check = [&]() {
4517 // Look for: ~x | Mask == -1
4518 if (isMaskOrZero(M, /*Not=*/false, Q)) {
4519 if (Value *NotX =
4520 IC.getFreelyInverted(X, X->hasOneUse(), &IC.Builder)) {
4521 X = NotX;
4522 return true;
4523 }
4524 }
4525 return false;
4526 };
4527 if (Check())
4528 return true;
4529 std::swap(X, M);
4530 return Check();
4531 }
4532 if (ICmpInst::isEquality(Pred) && match(Op1, m_Zero()) &&
4533 match(Op0, m_OneUse(m_And(m_Value(X), m_Value(M))))) {
4534 auto Check = [&]() {
4535 // Look for: x & ~Mask == 0
4536 if (isMaskOrZero(M, /*Not=*/true, Q)) {
4537 if (Value *NotM =
4538 IC.getFreelyInverted(M, M->hasOneUse(), &IC.Builder)) {
4539 M = NotM;
4540 return true;
4541 }
4542 }
4543 return false;
4544 };
4545 if (Check())
4546 return true;
4547 std::swap(X, M);
4548 return Check();
4549 }
4550 return false;
4551 };
4552
4553 if (!IsLowBitMask())
4554 return nullptr;
4555
4556 return IC.Builder.CreateICmp(DstPred, X, M);
4557}
4558
4559/// Some comparisons can be simplified.
4560/// In this case, we are looking for comparisons that look like
4561/// a check for a lossy signed truncation.
4562/// Folds: (MaskedBits is a constant.)
4563/// ((%x << MaskedBits) a>> MaskedBits) SrcPred %x
4564/// Into:
4565/// (add %x, (1 << (KeptBits-1))) DstPred (1 << KeptBits)
4566/// Where KeptBits = bitwidth(%x) - MaskedBits
4567static Value *
4569 InstCombiner::BuilderTy &Builder) {
4570 CmpPredicate SrcPred;
4571 Value *X;
4572 const APInt *C0, *C1; // FIXME: non-splats, potentially with undef.
4573 // We are ok with 'shl' having multiple uses, but 'ashr' must be one-use.
4574 if (!match(&I, m_c_ICmp(SrcPred,
4576 m_APInt(C1))),
4577 m_Deferred(X))))
4578 return nullptr;
4579
4580 // Potential handling of non-splats: for each element:
4581 // * if both are undef, replace with constant 0.
4582 // Because (1<<0) is OK and is 1, and ((1<<0)>>1) is also OK and is 0.
4583 // * if both are not undef, and are different, bailout.
4584 // * else, only one is undef, then pick the non-undef one.
4585
4586 // The shift amount must be equal.
4587 if (*C0 != *C1)
4588 return nullptr;
4589 const APInt &MaskedBits = *C0;
4590 assert(MaskedBits != 0 && "shift by zero should be folded away already.");
4591
4592 ICmpInst::Predicate DstPred;
4593 switch (SrcPred) {
4595 // ((%x << MaskedBits) a>> MaskedBits) == %x
4596 // =>
4597 // (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits)
4599 break;
4601 // ((%x << MaskedBits) a>> MaskedBits) != %x
4602 // =>
4603 // (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits)
4605 break;
4606 // FIXME: are more folds possible?
4607 default:
4608 return nullptr;
4609 }
4610
4611 auto *XType = X->getType();
4612 const unsigned XBitWidth = XType->getScalarSizeInBits();
4613 const APInt BitWidth = APInt(XBitWidth, XBitWidth);
4614 assert(BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched");
4615
4616 // KeptBits = bitwidth(%x) - MaskedBits
4617 const APInt KeptBits = BitWidth - MaskedBits;
4618 assert(KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable");
4619 // ICmpCst = (1 << KeptBits)
4620 const APInt ICmpCst = APInt(XBitWidth, 1).shl(KeptBits);
4621 assert(ICmpCst.isPowerOf2());
4622 // AddCst = (1 << (KeptBits-1))
4623 const APInt AddCst = ICmpCst.lshr(1);
4624 assert(AddCst.ult(ICmpCst) && AddCst.isPowerOf2());
4625
4626 // T0 = add %x, AddCst
4627 Value *T0 = Builder.CreateAdd(X, ConstantInt::get(XType, AddCst));
4628 // T1 = T0 DstPred ICmpCst
4629 Value *T1 = Builder.CreateICmp(DstPred, T0, ConstantInt::get(XType, ICmpCst));
4630
4631 return T1;
4632}
4633
4634// Given pattern:
4635// icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
4636// we should move shifts to the same hand of 'and', i.e. rewrite as
4637// icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x)
4638// We are only interested in opposite logical shifts here.
4639// One of the shifts can be truncated.
4640// If we can, we want to end up creating 'lshr' shift.
4641static Value *
4643 InstCombiner::BuilderTy &Builder) {
4644 if (!I.isEquality() || !match(I.getOperand(1), m_Zero()) ||
4645 !I.getOperand(0)->hasOneUse())
4646 return nullptr;
4647
4648 auto m_AnyLogicalShift = m_LogicalShift(m_Value(), m_Value());
4649
4650 // Look for an 'and' of two logical shifts, one of which may be truncated.
4651 // We use m_TruncOrSelf() on the RHS to correctly handle commutative case.
4652 Instruction *XShift, *MaybeTruncation, *YShift;
4653 if (!match(
4654 I.getOperand(0),
4655 m_c_And(m_CombineAnd(m_AnyLogicalShift, m_Instruction(XShift)),
4657 m_AnyLogicalShift, m_Instruction(YShift))),
4658 m_Instruction(MaybeTruncation)))))
4659 return nullptr;
4660
4661 // We potentially looked past 'trunc', but only when matching YShift,
4662 // therefore YShift must have the widest type.
4663 Instruction *WidestShift = YShift;
4664 // Therefore XShift must have the shallowest type.
4665 // Or they both have identical types if there was no truncation.
4666 Instruction *NarrowestShift = XShift;
4667
4668 Type *WidestTy = WidestShift->getType();
4669 Type *NarrowestTy = NarrowestShift->getType();
4670 assert(NarrowestTy == I.getOperand(0)->getType() &&
4671 "We did not look past any shifts while matching XShift though.");
4672 bool HadTrunc = WidestTy != I.getOperand(0)->getType();
4673
4674 // If YShift is a 'lshr', swap the shifts around.
4675 if (match(YShift, m_LShr(m_Value(), m_Value())))
4676 std::swap(XShift, YShift);
4677
4678 // The shifts must be in opposite directions.
4679 auto XShiftOpcode = XShift->getOpcode();
4680 if (XShiftOpcode == YShift->getOpcode())
4681 return nullptr; // Do not care about same-direction shifts here.
4682
4683 Value *X, *XShAmt, *Y, *YShAmt;
4684 match(XShift, m_BinOp(m_Value(X), m_ZExtOrSelf(m_Value(XShAmt))));
4685 match(YShift, m_BinOp(m_Value(Y), m_ZExtOrSelf(m_Value(YShAmt))));
4686
4687 // If one of the values being shifted is a constant, then we will end with
4688 // and+icmp, and [zext+]shift instrs will be constant-folded. If they are not,
4689 // however, we will need to ensure that we won't increase instruction count.
4690 if (!isa<Constant>(X) && !isa<Constant>(Y)) {
4691 // At least one of the hands of the 'and' should be one-use shift.
4692 if (!match(I.getOperand(0),
4693 m_c_And(m_OneUse(m_AnyLogicalShift), m_Value())))
4694 return nullptr;
4695 if (HadTrunc) {
4696 // Due to the 'trunc', we will need to widen X. For that either the old
4697 // 'trunc' or the shift amt in the non-truncated shift should be one-use.
4698 if (!MaybeTruncation->hasOneUse() &&
4699 !NarrowestShift->getOperand(1)->hasOneUse())
4700 return nullptr;
4701 }
4702 }
4703
4704 // We have two shift amounts from two different shifts. The types of those
4705 // shift amounts may not match. If that's the case let's bailout now.
4706 if (XShAmt->getType() != YShAmt->getType())
4707 return nullptr;
4708
4709 // As input, we have the following pattern:
4710 // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
4711 // We want to rewrite that as:
4712 // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x)
4713 // While we know that originally (Q+K) would not overflow
4714 // (because 2 * (N-1) u<= iN -1), we have looked past extensions of
4715 // shift amounts. so it may now overflow in smaller bitwidth.
4716 // To ensure that does not happen, we need to ensure that the total maximal
4717 // shift amount is still representable in that smaller bit width.
4718 unsigned MaximalPossibleTotalShiftAmount =
4719 (WidestTy->getScalarSizeInBits() - 1) +
4720 (NarrowestTy->getScalarSizeInBits() - 1);
4721 APInt MaximalRepresentableShiftAmount =
4723 if (MaximalRepresentableShiftAmount.ult(MaximalPossibleTotalShiftAmount))
4724 return nullptr;
4725
4726 // Can we fold (XShAmt+YShAmt) ?
4727 auto *NewShAmt = dyn_cast_or_null<Constant>(
4728 simplifyAddInst(XShAmt, YShAmt, /*isNSW=*/false,
4729 /*isNUW=*/false, SQ.getWithInstruction(&I)));
4730 if (!NewShAmt)
4731 return nullptr;
4732 if (NewShAmt->getType() != WidestTy) {
4733 NewShAmt =
4734 ConstantFoldCastOperand(Instruction::ZExt, NewShAmt, WidestTy, SQ.DL);
4735 if (!NewShAmt)
4736 return nullptr;
4737 }
4738 unsigned WidestBitWidth = WidestTy->getScalarSizeInBits();
4739
4740 // Is the new shift amount smaller than the bit width?
4741 // FIXME: could also rely on ConstantRange.
4742 if (!match(NewShAmt,
4744 APInt(WidestBitWidth, WidestBitWidth))))
4745 return nullptr;
4746
4747 // An extra legality check is needed if we had trunc-of-lshr.
4748 if (HadTrunc && match(WidestShift, m_LShr(m_Value(), m_Value()))) {
4749 auto CanFold = [NewShAmt, WidestBitWidth, NarrowestShift, SQ,
4750 WidestShift]() {
4751 // It isn't obvious whether it's worth it to analyze non-constants here.
4752 // Also, let's basically give up on non-splat cases, pessimizing vectors.
4753 // If *any* of these preconditions matches we can perform the fold.
4754 Constant *NewShAmtSplat = NewShAmt->getType()->isVectorTy()
4755 ? NewShAmt->getSplatValue()
4756 : NewShAmt;
4757 // If it's edge-case shift (by 0 or by WidestBitWidth-1) we can fold.
4758 if (NewShAmtSplat &&
4759 (NewShAmtSplat->isNullValue() ||
4760 NewShAmtSplat->getUniqueInteger() == WidestBitWidth - 1))
4761 return true;
4762 // We consider *min* leading zeros so a single outlier
4763 // blocks the transform as opposed to allowing it.
4764 if (auto *C = dyn_cast<Constant>(NarrowestShift->getOperand(0))) {
4765 KnownBits Known = computeKnownBits(C, SQ.DL);
4766 unsigned MinLeadZero = Known.countMinLeadingZeros();
4767 // If the value being shifted has at most lowest bit set we can fold.
4768 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero;
4769 if (MaxActiveBits <= 1)
4770 return true;
4771 // Precondition: NewShAmt u<= countLeadingZeros(C)
4772 if (NewShAmtSplat && NewShAmtSplat->getUniqueInteger().ule(MinLeadZero))
4773 return true;
4774 }
4775 if (auto *C = dyn_cast<Constant>(WidestShift->getOperand(0))) {
4776 KnownBits Known = computeKnownBits(C, SQ.DL);
4777 unsigned MinLeadZero = Known.countMinLeadingZeros();
4778 // If the value being shifted has at most lowest bit set we can fold.
4779 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero;
4780 if (MaxActiveBits <= 1)
4781 return true;
4782 // Precondition: ((WidestBitWidth-1)-NewShAmt) u<= countLeadingZeros(C)
4783 if (NewShAmtSplat) {
4784 APInt AdjNewShAmt =
4785 (WidestBitWidth - 1) - NewShAmtSplat->getUniqueInteger();
4786 if (AdjNewShAmt.ule(MinLeadZero))
4787 return true;
4788 }
4789 }
4790 return false; // Can't tell if it's ok.
4791 };
4792 if (!CanFold())
4793 return nullptr;
4794 }
4795
4796 // All good, we can do this fold.
4797 X = Builder.CreateZExt(X, WidestTy);
4798 Y = Builder.CreateZExt(Y, WidestTy);
4799 // The shift is the same that was for X.
4800 Value *T0 = XShiftOpcode == Instruction::BinaryOps::LShr
4801 ? Builder.CreateLShr(X, NewShAmt)
4802 : Builder.CreateShl(X, NewShAmt);
4803 Value *T1 = Builder.CreateAnd(T0, Y);
4804 return Builder.CreateICmp(I.getPredicate(), T1,
4805 Constant::getNullValue(WidestTy));
4806}
4807
4808/// Fold
4809/// (-1 u/ x) u< y
4810/// ((x * y) ?/ x) != y
4811/// to
4812/// @llvm.?mul.with.overflow(x, y) plus extraction of overflow bit
4813/// Note that the comparison is commutative, while inverted (u>=, ==) predicate
4814/// will mean that we are looking for the opposite answer.
4816 CmpPredicate Pred;
4817 Value *X, *Y;
4819 Instruction *Div;
4820 bool NeedNegation;
4821 // Look for: (-1 u/ x) u</u>= y
4822 if (!I.isEquality() &&
4823 match(&I, m_c_ICmp(Pred,
4825 m_Instruction(Div)),
4826 m_Value(Y)))) {
4827 Mul = nullptr;
4828
4829 // Are we checking that overflow does not happen, or does happen?
4830 switch (Pred) {
4832 NeedNegation = false;
4833 break; // OK
4835 NeedNegation = true;
4836 break; // OK
4837 default:
4838 return nullptr; // Wrong predicate.
4839 }
4840 } else // Look for: ((x * y) / x) !=/== y
4841 if (I.isEquality() &&
4842 match(&I,
4843 m_c_ICmp(Pred, m_Value(Y),
4846 m_Value(X)),
4848 m_Deferred(X))),
4849 m_Instruction(Div))))) {
4850 NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ;
4851 } else
4852 return nullptr;
4853
4855 // If the pattern included (x * y), we'll want to insert new instructions
4856 // right before that original multiplication so that we can replace it.
4857 bool MulHadOtherUses = Mul && !Mul->hasOneUse();
4858 if (MulHadOtherUses)
4860
4862 Div->getOpcode() == Instruction::UDiv ? Intrinsic::umul_with_overflow
4863 : Intrinsic::smul_with_overflow,
4864 X->getType(), {X, Y}, /*FMFSource=*/nullptr, "mul");
4865
4866 // If the multiplication was used elsewhere, to ensure that we don't leave
4867 // "duplicate" instructions, replace uses of that original multiplication
4868 // with the multiplication result from the with.overflow intrinsic.
4869 if (MulHadOtherUses)
4870 replaceInstUsesWith(*Mul, Builder.CreateExtractValue(Call, 0, "mul.val"));
4871
4872 Value *Res = Builder.CreateExtractValue(Call, 1, "mul.ov");
4873 if (NeedNegation) // This technically increases instruction count.
4874 Res = Builder.CreateNot(Res, "mul.not.ov");
4875
4876 // If we replaced the mul, erase it. Do this after all uses of Builder,
4877 // as the mul is used as insertion point.
4878 if (MulHadOtherUses)
4880
4881 return Res;
4882}
4883
4885 InstCombiner::BuilderTy &Builder) {
4886 CmpPredicate Pred;
4887 Value *X;
4888 if (match(&I, m_c_ICmp(Pred, m_NSWNeg(m_Value(X)), m_Deferred(X)))) {
4889
4890 if (ICmpInst::isSigned(Pred))
4891 Pred = ICmpInst::getSwappedPredicate(Pred);
4892 else if (ICmpInst::isUnsigned(Pred))
4893 Pred = ICmpInst::getSignedPredicate(Pred);
4894 // else for equality-comparisons just keep the predicate.
4895
4896 return ICmpInst::Create(Instruction::ICmp, Pred, X,
4897 Constant::getNullValue(X->getType()), I.getName());
4898 }
4899
4900 // A value is not equal to its negation unless that value is 0 or
4901 // MinSignedValue, ie: a != -a --> (a & MaxSignedVal) != 0
4902 if (match(&I, m_c_ICmp(Pred, m_OneUse(m_Neg(m_Value(X))), m_Deferred(X))) &&
4903 ICmpInst::isEquality(Pred)) {
4904 Type *Ty = X->getType();
4906 Constant *MaxSignedVal =
4907 ConstantInt::get(Ty, APInt::getSignedMaxValue(BitWidth));
4908 Value *And = Builder.CreateAnd(X, MaxSignedVal);
4909 Constant *Zero = Constant::getNullValue(Ty);
4910 return CmpInst::Create(Instruction::ICmp, Pred, And, Zero);
4911 }
4912
4913 return nullptr;
4914}
4915
4917 InstCombinerImpl &IC) {
4918 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *A;
4919 // Normalize and operand as operand 0.
4920 CmpInst::Predicate Pred = I.getPredicate();
4921 if (match(Op1, m_c_And(m_Specific(Op0), m_Value()))) {
4922 std::swap(Op0, Op1);
4923 Pred = ICmpInst::getSwappedPredicate(Pred);
4924 }
4925
4926 if (!match(Op0, m_c_And(m_Specific(Op1), m_Value(A))))
4927 return nullptr;
4928
4929 // (icmp (X & Y) u< X --> (X & Y) != X
4930 if (Pred == ICmpInst::ICMP_ULT)
4931 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
4932
4933 // (icmp (X & Y) u>= X --> (X & Y) == X
4934 if (Pred == ICmpInst::ICMP_UGE)
4935 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
4936
4937 if (ICmpInst::isEquality(Pred) && Op0->hasOneUse()) {
4938 // icmp (X & Y) eq/ne Y --> (X | ~Y) eq/ne -1 if Y is freely invertible and
4939 // Y is non-constant. If Y is constant the `X & C == C` form is preferable
4940 // so don't do this fold.
4941 if (!match(Op1, m_ImmConstant()))
4942 if (auto *NotOp1 =
4943 IC.getFreelyInverted(Op1, !Op1->hasNUsesOrMore(3), &IC.Builder))
4944 return new ICmpInst(Pred, IC.Builder.CreateOr(A, NotOp1),
4945 Constant::getAllOnesValue(Op1->getType()));
4946 // icmp (X & Y) eq/ne Y --> (~X & Y) eq/ne 0 if X is freely invertible.
4947 if (auto *NotA = IC.getFreelyInverted(A, A->hasOneUse(), &IC.Builder))
4948 return new ICmpInst(Pred, IC.Builder.CreateAnd(Op1, NotA),
4949 Constant::getNullValue(Op1->getType()));
4950 }
4951
4952 if (!ICmpInst::isSigned(Pred))
4953 return nullptr;
4954
4955 KnownBits KnownY = IC.computeKnownBits(A, /*Depth=*/0, &I);
4956 // (X & NegY) spred X --> (X & NegY) upred X
4957 if (KnownY.isNegative())
4958 return new ICmpInst(ICmpInst::getUnsignedPredicate(Pred), Op0, Op1);
4959
4960 if (Pred != ICmpInst::ICMP_SLE && Pred != ICmpInst::ICMP_SGT)
4961 return nullptr;
4962
4963 if (KnownY.isNonNegative())
4964 // (X & PosY) s<= X --> X s>= 0
4965 // (X & PosY) s> X --> X s< 0
4966 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
4967 Constant::getNullValue(Op1->getType()));
4968
4970 // (NegX & Y) s<= NegX --> Y s< 0
4971 // (NegX & Y) s> NegX --> Y s>= 0
4973 Constant::getNullValue(A->getType()));
4974
4975 return nullptr;
4976}
4977
4979 InstCombinerImpl &IC) {
4980 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *A;
4981
4982 // Normalize or operand as operand 0.
4983 CmpInst::Predicate Pred = I.getPredicate();
4984 if (match(Op1, m_c_Or(m_Specific(Op0), m_Value(A)))) {
4985 std::swap(Op0, Op1);
4986 Pred = ICmpInst::getSwappedPredicate(Pred);
4987 } else if (!match(Op0, m_c_Or(m_Specific(Op1), m_Value(A)))) {
4988 return nullptr;
4989 }
4990
4991 // icmp (X | Y) u<= X --> (X | Y) == X
4992 if (Pred == ICmpInst::ICMP_ULE)
4993 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
4994
4995 // icmp (X | Y) u> X --> (X | Y) != X
4996 if (Pred == ICmpInst::ICMP_UGT)
4997 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
4998
4999 if (ICmpInst::isEquality(Pred) && Op0->hasOneUse()) {
5000 // icmp (X | Y) eq/ne Y --> (X & ~Y) eq/ne 0 if Y is freely invertible
5001 if (Value *NotOp1 =
5002 IC.getFreelyInverted(Op1, !Op1->hasNUsesOrMore(3), &IC.Builder))
5003 return new ICmpInst(Pred, IC.Builder.CreateAnd(A, NotOp1),
5004 Constant::getNullValue(Op1->getType()));
5005 // icmp (X | Y) eq/ne Y --> (~X | Y) eq/ne -1 if X is freely invertible.
5006 if (Value *NotA = IC.getFreelyInverted(A, A->hasOneUse(), &IC.Builder))
5007 return new ICmpInst(Pred, IC.Builder.CreateOr(Op1, NotA),
5008 Constant::getAllOnesValue(Op1->getType()));
5009 }
5010 return nullptr;
5011}
5012
5014 InstCombinerImpl &IC) {
5015 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1), *A;
5016 // Normalize xor operand as operand 0.
5017 CmpInst::Predicate Pred = I.getPredicate();
5018 if (match(Op1, m_c_Xor(m_Specific(Op0), m_Value()))) {
5019 std::swap(Op0, Op1);
5020 Pred = ICmpInst::getSwappedPredicate(Pred);
5021 }
5022 if (!match(Op0, m_c_Xor(m_Specific(Op1), m_Value(A))))
5023 return nullptr;
5024
5025 // icmp (X ^ Y_NonZero) u>= X --> icmp (X ^ Y_NonZero) u> X
5026 // icmp (X ^ Y_NonZero) u<= X --> icmp (X ^ Y_NonZero) u< X
5027 // icmp (X ^ Y_NonZero) s>= X --> icmp (X ^ Y_NonZero) s> X
5028 // icmp (X ^ Y_NonZero) s<= X --> icmp (X ^ Y_NonZero) s< X
5030 if (PredOut != Pred && isKnownNonZero(A, Q))
5031 return new ICmpInst(PredOut, Op0, Op1);
5032
5033 return nullptr;
5034}
5035
5036/// Try to fold icmp (binop), X or icmp X, (binop).
5037/// TODO: A large part of this logic is duplicated in InstSimplify's
5038/// simplifyICmpWithBinOp(). We should be able to share that and avoid the code
5039/// duplication.
5041 const SimplifyQuery &SQ) {
5043 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
5044
5045 // Special logic for binary operators.
5046 BinaryOperator *BO0 = dyn_cast<BinaryOperator>(Op0);
5047 BinaryOperator *BO1 = dyn_cast<BinaryOperator>(Op1);
5048 if (!BO0 && !BO1)
5049 return nullptr;
5050
5051 if (Instruction *NewICmp = foldICmpXNegX(I, Builder))
5052 return NewICmp;
5053
5054 const CmpInst::Predicate Pred = I.getPredicate();
5055 Value *X;
5056
5057 // Convert add-with-unsigned-overflow comparisons into a 'not' with compare.
5058 // (Op1 + X) u</u>= Op1 --> ~Op1 u</u>= X
5059 if (match(Op0, m_OneUse(m_c_Add(m_Specific(Op1), m_Value(X)))) &&
5060 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
5061 return new ICmpInst(Pred, Builder.CreateNot(Op1), X);
5062 // Op0 u>/u<= (Op0 + X) --> X u>/u<= ~Op0
5063 if (match(Op1, m_OneUse(m_c_Add(m_Specific(Op0), m_Value(X)))) &&
5064 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE))
5065 return new ICmpInst(Pred, X, Builder.CreateNot(Op0));
5066
5067 {
5068 // (Op1 + X) + C u</u>= Op1 --> ~C - X u</u>= Op1
5069 Constant *C;
5070 if (match(Op0, m_OneUse(m_Add(m_c_Add(m_Specific(Op1), m_Value(X)),
5071 m_ImmConstant(C)))) &&
5072 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) {
5074 return new ICmpInst(Pred, Builder.CreateSub(C2, X), Op1);
5075 }
5076 // Op0 u>/u<= (Op0 + X) + C --> Op0 u>/u<= ~C - X
5077 if (match(Op1, m_OneUse(m_Add(m_c_Add(m_Specific(Op0), m_Value(X)),
5078 m_ImmConstant(C)))) &&
5079 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) {
5081 return new ICmpInst(Pred, Op0, Builder.CreateSub(C2, X));
5082 }
5083 }
5084
5085 // (icmp eq/ne (X, -P2), INT_MIN)
5086 // -> (icmp slt/sge X, INT_MIN + P2)
5087 if (ICmpInst::isEquality(Pred) && BO0 &&
5088 match(I.getOperand(1), m_SignMask()) &&
5090 // Will Constant fold.
5091 Value *NewC = Builder.CreateSub(I.getOperand(1), BO0->getOperand(1));
5092 return new ICmpInst(Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_SLT
5094 BO0->getOperand(0), NewC);
5095 }
5096
5097 {
5098 // Similar to above: an unsigned overflow comparison may use offset + mask:
5099 // ((Op1 + C) & C) u< Op1 --> Op1 != 0
5100 // ((Op1 + C) & C) u>= Op1 --> Op1 == 0
5101 // Op0 u> ((Op0 + C) & C) --> Op0 != 0
5102 // Op0 u<= ((Op0 + C) & C) --> Op0 == 0
5103 BinaryOperator *BO;
5104 const APInt *C;
5105 if ((Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) &&
5106 match(Op0, m_And(m_BinOp(BO), m_LowBitMask(C))) &&
5108 CmpInst::Predicate NewPred =
5110 Constant *Zero = ConstantInt::getNullValue(Op1->getType());
5111 return new ICmpInst(NewPred, Op1, Zero);
5112 }
5113
5114 if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) &&
5115 match(Op1, m_And(m_BinOp(BO), m_LowBitMask(C))) &&
5117 CmpInst::Predicate NewPred =
5119 Constant *Zero = ConstantInt::getNullValue(Op1->getType());
5120 return new ICmpInst(NewPred, Op0, Zero);
5121 }
5122 }
5123
5124 bool NoOp0WrapProblem = false, NoOp1WrapProblem = false;
5125 bool Op0HasNUW = false, Op1HasNUW = false;
5126 bool Op0HasNSW = false, Op1HasNSW = false;
5127 // Analyze the case when either Op0 or Op1 is an add instruction.
5128 // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null).
5129 auto hasNoWrapProblem = [](const BinaryOperator &BO, CmpInst::Predicate Pred,
5130 bool &HasNSW, bool &HasNUW) -> bool {
5131 if (isa<OverflowingBinaryOperator>(BO)) {
5132 HasNUW = BO.hasNoUnsignedWrap();
5133 HasNSW = BO.hasNoSignedWrap();
5134 return ICmpInst::isEquality(Pred) ||
5135 (CmpInst::isUnsigned(Pred) && HasNUW) ||
5136 (CmpInst::isSigned(Pred) && HasNSW);
5137 } else if (BO.getOpcode() == Instruction::Or) {
5138 HasNUW = true;
5139 HasNSW = true;
5140 return true;
5141 } else {
5142 return false;
5143 }
5144 };
5145 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
5146
5147 if (BO0) {
5148 match(BO0, m_AddLike(m_Value(A), m_Value(B)));
5149 NoOp0WrapProblem = hasNoWrapProblem(*BO0, Pred, Op0HasNSW, Op0HasNUW);
5150 }
5151 if (BO1) {
5152 match(BO1, m_AddLike(m_Value(C), m_Value(D)));
5153 NoOp1WrapProblem = hasNoWrapProblem(*BO1, Pred, Op1HasNSW, Op1HasNUW);
5154 }
5155
5156 // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow.
5157 // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow.
5158 if ((A == Op1 || B == Op1) && NoOp0WrapProblem)
5159 return new ICmpInst(Pred, A == Op1 ? B : A,
5160 Constant::getNullValue(Op1->getType()));
5161
5162 // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow.
5163 // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow.
5164 if ((C == Op0 || D == Op0) && NoOp1WrapProblem)
5165 return new ICmpInst(Pred, Constant::getNullValue(Op0->getType()),
5166 C == Op0 ? D : C);
5167
5168 // icmp (A+B), (A+D) -> icmp B, D for equalities or if there is no overflow.
5169 if (A && C && (A == C || A == D || B == C || B == D) && NoOp0WrapProblem &&
5170 NoOp1WrapProblem) {
5171 // Determine Y and Z in the form icmp (X+Y), (X+Z).
5172 Value *Y, *Z;
5173 if (A == C) {
5174 // C + B == C + D -> B == D
5175 Y = B;
5176 Z = D;
5177 } else if (A == D) {
5178 // D + B == C + D -> B == C
5179 Y = B;
5180 Z = C;
5181 } else if (B == C) {
5182 // A + C == C + D -> A == D
5183 Y = A;
5184 Z = D;
5185 } else {
5186 assert(B == D);
5187 // A + D == C + D -> A == C
5188 Y = A;
5189 Z = C;
5190 }
5191 return new ICmpInst(Pred, Y, Z);
5192 }
5193
5194 // icmp slt (A + -1), Op1 -> icmp sle A, Op1
5195 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT &&
5196 match(B, m_AllOnes()))
5197 return new ICmpInst(CmpInst::ICMP_SLE, A, Op1);
5198
5199 // icmp sge (A + -1), Op1 -> icmp sgt A, Op1
5200 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE &&
5201 match(B, m_AllOnes()))
5202 return new ICmpInst(CmpInst::ICMP_SGT, A, Op1);
5203
5204 // icmp sle (A + 1), Op1 -> icmp slt A, Op1
5205 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(B, m_One()))
5206 return new ICmpInst(CmpInst::ICMP_SLT, A, Op1);
5207
5208 // icmp sgt (A + 1), Op1 -> icmp sge A, Op1
5209 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(B, m_One()))
5210 return new ICmpInst(CmpInst::ICMP_SGE, A, Op1);
5211
5212 // icmp sgt Op0, (C + -1) -> icmp sge Op0, C
5213 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT &&
5214 match(D, m_AllOnes()))
5215 return new ICmpInst(CmpInst::ICMP_SGE, Op0, C);
5216
5217 // icmp sle Op0, (C + -1) -> icmp slt Op0, C
5218 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE &&
5219 match(D, m_AllOnes()))
5220 return new ICmpInst(CmpInst::ICMP_SLT, Op0, C);
5221
5222 // icmp sge Op0, (C + 1) -> icmp sgt Op0, C
5223 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(D, m_One()))
5224 return new ICmpInst(CmpInst::ICMP_SGT, Op0, C);
5225
5226 // icmp slt Op0, (C + 1) -> icmp sle Op0, C
5227 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(D, m_One()))
5228 return new ICmpInst(CmpInst::ICMP_SLE, Op0, C);
5229
5230 // TODO: The subtraction-related identities shown below also hold, but
5231 // canonicalization from (X -nuw 1) to (X + -1) means that the combinations
5232 // wouldn't happen even if they were implemented.
5233 //
5234 // icmp ult (A - 1), Op1 -> icmp ule A, Op1
5235 // icmp uge (A - 1), Op1 -> icmp ugt A, Op1
5236 // icmp ugt Op0, (C - 1) -> icmp uge Op0, C
5237 // icmp ule Op0, (C - 1) -> icmp ult Op0, C
5238
5239 // icmp ule (A + 1), Op0 -> icmp ult A, Op1
5240 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(B, m_One()))
5241 return new ICmpInst(CmpInst::ICMP_ULT, A, Op1);
5242
5243 // icmp ugt (A + 1), Op0 -> icmp uge A, Op1
5244 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(B, m_One()))
5245 return new ICmpInst(CmpInst::ICMP_UGE, A, Op1);
5246
5247 // icmp uge Op0, (C + 1) -> icmp ugt Op0, C
5248 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(D, m_One()))
5249 return new ICmpInst(CmpInst::ICMP_UGT, Op0, C);
5250
5251 // icmp ult Op0, (C + 1) -> icmp ule Op0, C
5252 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(D, m_One()))
5253 return new ICmpInst(CmpInst::ICMP_ULE, Op0, C);
5254
5255 // if C1 has greater magnitude than C2:
5256 // icmp (A + C1), (C + C2) -> icmp (A + C3), C
5257 // s.t. C3 = C1 - C2
5258 //
5259 // if C2 has greater magnitude than C1:
5260 // icmp (A + C1), (C + C2) -> icmp A, (C + C3)
5261 // s.t. C3 = C2 - C1
5262 if (A && C && NoOp0WrapProblem && NoOp1WrapProblem &&
5263 (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) {
5264 const APInt *AP1, *AP2;
5265 // TODO: Support non-uniform vectors.
5266 // TODO: Allow poison passthrough if B or D's element is poison.
5267 if (match(B, m_APIntAllowPoison(AP1)) &&
5268 match(D, m_APIntAllowPoison(AP2)) &&
5269 AP1->isNegative() == AP2->isNegative()) {
5270 APInt AP1Abs = AP1->abs();
5271 APInt AP2Abs = AP2->abs();
5272 if (AP1Abs.uge(AP2Abs)) {
5273 APInt Diff = *AP1 - *AP2;
5274 Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff);
5275 Value *NewAdd = Builder.CreateAdd(
5276 A, C3, "", Op0HasNUW && Diff.ule(*AP1), Op0HasNSW);
5277 return new ICmpInst(Pred, NewAdd, C);
5278 } else {
5279 APInt Diff = *AP2 - *AP1;
5280 Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff);
5281 Value *NewAdd = Builder.CreateAdd(
5282 C, C3, "", Op1HasNUW && Diff.ule(*AP2), Op1HasNSW);
5283 return new ICmpInst(Pred, A, NewAdd);
5284 }
5285 }
5286 Constant *Cst1, *Cst2;
5287 if (match(B, m_ImmConstant(Cst1)) && match(D, m_ImmConstant(Cst2)) &&
5288 ICmpInst::isEquality(Pred)) {
5289 Constant *Diff = ConstantExpr::getSub(Cst2, Cst1);
5290 Value *NewAdd = Builder.CreateAdd(C, Diff);
5291 return new ICmpInst(Pred, A, NewAdd);
5292 }
5293 }
5294
5295 // Analyze the case when either Op0 or Op1 is a sub instruction.
5296 // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null).
5297 A = nullptr;
5298 B = nullptr;
5299 C = nullptr;
5300 D = nullptr;
5301 if (BO0 && BO0->getOpcode() == Instruction::Sub) {
5302 A = BO0->getOperand(0);
5303 B = BO0->getOperand(1);
5304 }
5305 if (BO1 && BO1->getOpcode() == Instruction::Sub) {
5306 C = BO1->getOperand(0);
5307 D = BO1->getOperand(1);
5308 }
5309
5310 // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow.
5311 if (A == Op1 && NoOp0WrapProblem)
5312 return new ICmpInst(Pred, Constant::getNullValue(Op1->getType()), B);
5313 // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow.
5314 if (C == Op0 && NoOp1WrapProblem)
5315 return new ICmpInst(Pred, D, Constant::getNullValue(Op0->getType()));
5316
5317 // Convert sub-with-unsigned-overflow comparisons into a comparison of args.
5318 // (A - B) u>/u<= A --> B u>/u<= A
5319 if (A == Op1 && (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE))
5320 return new ICmpInst(Pred, B, A);
5321 // C u</u>= (C - D) --> C u</u>= D
5322 if (C == Op0 && (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
5323 return new ICmpInst(Pred, C, D);
5324 // (A - B) u>=/u< A --> B u>/u<= A iff B != 0
5325 if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
5326 isKnownNonZero(B, Q))
5328 // C u<=/u> (C - D) --> C u</u>= D iff B != 0
5329 if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
5330 isKnownNonZero(D, Q))
5332
5333 // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow.
5334 if (B && D && B == D && NoOp0WrapProblem && NoOp1WrapProblem)
5335 return new ICmpInst(Pred, A, C);
5336
5337 // icmp (A-B), (A-D) -> icmp D, B for equalities or if there is no overflow.
5338 if (A && C && A == C && NoOp0WrapProblem && NoOp1WrapProblem)
5339 return new ICmpInst(Pred, D, B);
5340
5341 // icmp (0-X) < cst --> x > -cst
5342 if (NoOp0WrapProblem && ICmpInst::isSigned(Pred)) {
5343 Value *X;
5344 if (match(BO0, m_Neg(m_Value(X))))
5345 if (Constant *RHSC = dyn_cast<Constant>(Op1))
5346 if (RHSC->isNotMinSignedValue())
5347 return new ICmpInst(I.getSwappedPredicate(), X,
5348 ConstantExpr::getNeg(RHSC));
5349 }
5350
5351 if (Instruction * R = foldICmpXorXX(I, Q, *this))
5352 return R;
5353 if (Instruction *R = foldICmpOrXX(I, Q, *this))
5354 return R;
5355
5356 {
5357 // Try to remove shared multiplier from comparison:
5358 // X * Z pred Y * Z
5359 Value *X, *Y, *Z;
5360 if ((match(Op0, m_Mul(m_Value(X), m_Value(Z))) &&
5361 match(Op1, m_c_Mul(m_Specific(Z), m_Value(Y)))) ||
5362 (match(Op0, m_Mul(m_Value(Z), m_Value(X))) &&
5363 match(Op1, m_c_Mul(m_Specific(Z), m_Value(Y))))) {
5364 if (ICmpInst::isSigned(Pred)) {
5365 if (Op0HasNSW && Op1HasNSW) {
5366 KnownBits ZKnown = computeKnownBits(Z, 0, &I);
5367 if (ZKnown.isStrictlyPositive())
5368 return new ICmpInst(Pred, X, Y);
5369 if (ZKnown.isNegative())
5370 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), X, Y);
5373 if (LessThan && match(LessThan, m_One()))
5374 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Z,
5375 Constant::getNullValue(Z->getType()));
5376 Value *GreaterThan = simplifyICmpInst(ICmpInst::ICMP_SGT, X, Y,
5378 if (GreaterThan && match(GreaterThan, m_One()))
5379 return new ICmpInst(Pred, Z, Constant::getNullValue(Z->getType()));
5380 }
5381 } else {
5382 bool NonZero;
5383 if (ICmpInst::isEquality(Pred)) {
5384 // If X != Y, fold (X *nw Z) eq/ne (Y *nw Z) -> Z eq/ne 0
5385 if (((Op0HasNSW && Op1HasNSW) || (Op0HasNUW && Op1HasNUW)) &&
5386 isKnownNonEqual(X, Y, DL, &AC, &I, &DT))
5387 return new ICmpInst(Pred, Z, Constant::getNullValue(Z->getType()));
5388
5389 KnownBits ZKnown = computeKnownBits(Z, 0, &I);
5390 // if Z % 2 != 0
5391 // X * Z eq/ne Y * Z -> X eq/ne Y
5392 if (ZKnown.countMaxTrailingZeros() == 0)
5393 return new ICmpInst(Pred, X, Y);
5394 NonZero = !ZKnown.One.isZero() || isKnownNonZero(Z, Q);
5395 // if Z != 0 and nsw(X * Z) and nsw(Y * Z)
5396 // X * Z eq/ne Y * Z -> X eq/ne Y
5397 if (NonZero && BO0 && BO1 && Op0HasNSW && Op1HasNSW)
5398 return new ICmpInst(Pred, X, Y);
5399 } else
5400 NonZero = isKnownNonZero(Z, Q);
5401
5402 // If Z != 0 and nuw(X * Z) and nuw(Y * Z)
5403 // X * Z u{lt/le/gt/ge}/eq/ne Y * Z -> X u{lt/le/gt/ge}/eq/ne Y
5404 if (NonZero && BO0 && BO1 && Op0HasNUW && Op1HasNUW)
5405 return new ICmpInst(Pred, X, Y);
5406 }
5407 }
5408 }
5409
5410 BinaryOperator *SRem = nullptr;
5411 // icmp (srem X, Y), Y
5412 if (BO0 && BO0->getOpcode() == Instruction::SRem && Op1 == BO0->getOperand(1))
5413 SRem = BO0;
5414 // icmp Y, (srem X, Y)
5415 else if (BO1 && BO1->getOpcode() == Instruction::SRem &&
5416 Op0 == BO1->getOperand(1))
5417 SRem = BO1;
5418 if (SRem) {
5419 // We don't check hasOneUse to avoid increasing register pressure because
5420 // the value we use is the same value this instruction was already using.
5421 switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(Pred) : Pred) {
5422 default:
5423 break;
5424 case ICmpInst::ICMP_EQ:
5425 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5426 case ICmpInst::ICMP_NE:
5427 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5428 case ICmpInst::ICMP_SGT:
5429 case ICmpInst::ICMP_SGE:
5430 return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(1),
5432 case ICmpInst::ICMP_SLT:
5433 case ICmpInst::ICMP_SLE:
5434 return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(1),
5436 }
5437 }
5438
5439 if (BO0 && BO1 && BO0->getOpcode() == BO1->getOpcode() &&
5440 (BO0->hasOneUse() || BO1->hasOneUse()) &&
5441 BO0->getOperand(1) == BO1->getOperand(1)) {
5442 switch (BO0->getOpcode()) {
5443 default:
5444 break;
5445 case Instruction::Add:
5446 case Instruction::Sub:
5447 case Instruction::Xor: {
5448 if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b
5449 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5450
5451 const APInt *C;
5452 if (match(BO0->getOperand(1), m_APInt(C))) {
5453 // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b
5454 if (C->isSignMask()) {
5455 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate();
5456 return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0));
5457 }
5458
5459 // icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b
5460 if (BO0->getOpcode() == Instruction::Xor && C->isMaxSignedValue()) {
5461 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate();
5462 NewPred = I.getSwappedPredicate(NewPred);
5463 return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0));
5464 }
5465 }
5466 break;
5467 }
5468 case Instruction::Mul: {
5469 if (!I.isEquality())
5470 break;
5471
5472 const APInt *C;
5473 if (match(BO0->getOperand(1), m_APInt(C)) && !C->isZero() &&
5474 !C->isOne()) {
5475 // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask)
5476 // Mask = -1 >> count-trailing-zeros(C).
5477 if (unsigned TZs = C->countr_zero()) {
5478 Constant *Mask = ConstantInt::get(
5479 BO0->getType(),
5480 APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs));
5481 Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask);
5482 Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask);
5483 return new ICmpInst(Pred, And1, And2);
5484 }
5485 }
5486 break;
5487 }
5488 case Instruction::UDiv:
5489 case Instruction::LShr:
5490 if (I.isSigned() || !BO0->isExact() || !BO1->isExact())
5491 break;
5492 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5493
5494 case Instruction::SDiv:
5495 if (!(I.isEquality() || match(BO0->getOperand(1), m_NonNegative())) ||
5496 !BO0->isExact() || !BO1->isExact())
5497 break;
5498 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5499
5500 case Instruction::AShr:
5501 if (!BO0->isExact() || !BO1->isExact())
5502 break;
5503 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5504
5505 case Instruction::Shl: {
5506 bool NUW = Op0HasNUW && Op1HasNUW;
5507 bool NSW = Op0HasNSW && Op1HasNSW;
5508 if (!NUW && !NSW)
5509 break;
5510 if (!NSW && I.isSigned())
5511 break;
5512 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
5513 }
5514 }
5515 }
5516
5517 if (BO0) {
5518 // Transform A & (L - 1) `ult` L --> L != 0
5519 auto LSubOne = m_Add(m_Specific(Op1), m_AllOnes());
5520 auto BitwiseAnd = m_c_And(m_Value(), LSubOne);
5521
5522 if (match(BO0, BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) {
5523 auto *Zero = Constant::getNullValue(BO0->getType());
5524 return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero);
5525 }
5526 }
5527
5528 // For unsigned predicates / eq / ne:
5529 // icmp pred (x << 1), x --> icmp getSignedPredicate(pred) x, 0
5530 // icmp pred x, (x << 1) --> icmp getSignedPredicate(pred) 0, x
5531 if (!ICmpInst::isSigned(Pred)) {
5532 if (match(Op0, m_Shl(m_Specific(Op1), m_One())))
5533 return new ICmpInst(ICmpInst::getSignedPredicate(Pred), Op1,
5534 Constant::getNullValue(Op1->getType()));
5535 else if (match(Op1, m_Shl(m_Specific(Op0), m_One())))
5536 return new ICmpInst(ICmpInst::getSignedPredicate(Pred),
5537 Constant::getNullValue(Op0->getType()), Op0);
5538 }
5539
5541 return replaceInstUsesWith(I, V);
5542
5543 if (Instruction *R = foldICmpAndXX(I, Q, *this))
5544 return R;
5545
5547 return replaceInstUsesWith(I, V);
5548
5550 return replaceInstUsesWith(I, V);
5551
5552 return nullptr;
5553}
5554
5555/// Fold icmp Pred min|max(X, Y), Z.
5558 Value *Z, CmpPredicate Pred) {
5559 Value *X = MinMax->getLHS();
5560 Value *Y = MinMax->getRHS();
5561 if (ICmpInst::isSigned(Pred) && !MinMax->isSigned())
5562 return nullptr;
5563 if (ICmpInst::isUnsigned(Pred) && MinMax->isSigned()) {
5564 // Revert the transform signed pred -> unsigned pred
5565 // TODO: We can flip the signedness of predicate if both operands of icmp
5566 // are negative.
5570 } else
5571 return nullptr;
5572 }
5574 auto IsCondKnownTrue = [](Value *Val) -> std::optional<bool> {
5575 if (!Val)
5576 return std::nullopt;
5577 if (match(Val, m_One()))
5578 return true;
5579 if (match(Val, m_Zero()))
5580 return false;
5581 return std::nullopt;
5582 };
5583 auto CmpXZ = IsCondKnownTrue(simplifyICmpInst(Pred, X, Z, Q));
5584 auto CmpYZ = IsCondKnownTrue(simplifyICmpInst(Pred, Y, Z, Q));
5585 if (!CmpXZ.has_value() && !CmpYZ.has_value())
5586 return nullptr;
5587 if (!CmpXZ.has_value()) {
5588 std::swap(X, Y);
5589 std::swap(CmpXZ, CmpYZ);
5590 }
5591
5592 auto FoldIntoCmpYZ = [&]() -> Instruction * {
5593 if (CmpYZ.has_value())
5594 return replaceInstUsesWith(I, ConstantInt::getBool(I.getType(), *CmpYZ));
5595 return ICmpInst::Create(Instruction::ICmp, Pred, Y, Z);
5596 };
5597
5598 switch (Pred) {
5599 case ICmpInst::ICMP_EQ:
5600 case ICmpInst::ICMP_NE: {
5601 // If X == Z:
5602 // Expr Result
5603 // min(X, Y) == Z X <= Y
5604 // max(X, Y) == Z X >= Y
5605 // min(X, Y) != Z X > Y
5606 // max(X, Y) != Z X < Y
5607 if ((Pred == ICmpInst::ICMP_EQ) == *CmpXZ) {
5608 ICmpInst::Predicate NewPred =
5609 ICmpInst::getNonStrictPredicate(MinMax->getPredicate());
5610 if (Pred == ICmpInst::ICMP_NE)
5611 NewPred = ICmpInst::getInversePredicate(NewPred);
5612 return ICmpInst::Create(Instruction::ICmp, NewPred, X, Y);
5613 }
5614 // Otherwise (X != Z):
5615 ICmpInst::Predicate NewPred = MinMax->getPredicate();
5616 auto MinMaxCmpXZ = IsCondKnownTrue(simplifyICmpInst(NewPred, X, Z, Q));
5617 if (!MinMaxCmpXZ.has_value()) {
5618 std::swap(X, Y);
5619 std::swap(CmpXZ, CmpYZ);
5620 // Re-check pre-condition X != Z
5621 if (!CmpXZ.has_value() || (Pred == ICmpInst::ICMP_EQ) == *CmpXZ)
5622 break;
5623 MinMaxCmpXZ = IsCondKnownTrue(simplifyICmpInst(NewPred, X, Z, Q));
5624 }
5625 if (!MinMaxCmpXZ.has_value())
5626 break;
5627 if (*MinMaxCmpXZ) {
5628 // Expr Fact Result
5629 // min(X, Y) == Z X < Z false
5630 // max(X, Y) == Z X > Z false
5631 // min(X, Y) != Z X < Z true
5632 // max(X, Y) != Z X > Z true
5633 return replaceInstUsesWith(
5634 I, ConstantInt::getBool(I.getType(), Pred == ICmpInst::ICMP_NE));
5635 } else {
5636 // Expr Fact Result
5637 // min(X, Y) == Z X > Z Y == Z
5638 // max(X, Y) == Z X < Z Y == Z
5639 // min(X, Y) != Z X > Z Y != Z
5640 // max(X, Y) != Z X < Z Y != Z
5641 return FoldIntoCmpYZ();
5642 }
5643 break;
5644 }
5645 case ICmpInst::ICMP_SLT:
5646 case ICmpInst::ICMP_ULT:
5647 case ICmpInst::ICMP_SLE:
5648 case ICmpInst::ICMP_ULE:
5649 case ICmpInst::ICMP_SGT:
5650 case ICmpInst::ICMP_UGT:
5651 case ICmpInst::ICMP_SGE:
5652 case ICmpInst::ICMP_UGE: {
5653 bool IsSame = MinMax->getPredicate() == ICmpInst::getStrictPredicate(Pred);
5654 if (*CmpXZ) {
5655 if (IsSame) {
5656 // Expr Fact Result
5657 // min(X, Y) < Z X < Z true
5658 // min(X, Y) <= Z X <= Z true
5659 // max(X, Y) > Z X > Z true
5660 // max(X, Y) >= Z X >= Z true
5661 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5662 } else {
5663 // Expr Fact Result
5664 // max(X, Y) < Z X < Z Y < Z
5665 // max(X, Y) <= Z X <= Z Y <= Z
5666 // min(X, Y) > Z X > Z Y > Z
5667 // min(X, Y) >= Z X >= Z Y >= Z
5668 return FoldIntoCmpYZ();
5669 }
5670 } else {
5671 if (IsSame) {
5672 // Expr Fact Result
5673 // min(X, Y) < Z X >= Z Y < Z
5674 // min(X, Y) <= Z X > Z Y <= Z
5675 // max(X, Y) > Z X <= Z Y > Z
5676 // max(X, Y) >= Z X < Z Y >= Z
5677 return FoldIntoCmpYZ();
5678 } else {
5679 // Expr Fact Result
5680 // max(X, Y) < Z X >= Z false
5681 // max(X, Y) <= Z X > Z false
5682 // min(X, Y) > Z X <= Z false
5683 // min(X, Y) >= Z X < Z false
5684 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5685 }
5686 }
5687 break;
5688 }
5689 default:
5690 break;
5691 }
5692
5693 return nullptr;
5694}
5695
5696// Canonicalize checking for a power-of-2-or-zero value:
5698 InstCombiner::BuilderTy &Builder) {
5699 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
5700 const CmpInst::Predicate Pred = I.getPredicate();
5701 Value *A = nullptr;
5702 bool CheckIs;
5703 if (I.isEquality()) {
5704 // (A & (A-1)) == 0 --> ctpop(A) < 2 (two commuted variants)
5705 // ((A-1) & A) != 0 --> ctpop(A) > 1 (two commuted variants)
5706 if (!match(Op0, m_OneUse(m_c_And(m_Add(m_Value(A), m_AllOnes()),
5707 m_Deferred(A)))) ||
5708 !match(Op1, m_ZeroInt()))
5709 A = nullptr;
5710
5711 // (A & -A) == A --> ctpop(A) < 2 (four commuted variants)
5712 // (-A & A) != A --> ctpop(A) > 1 (four commuted variants)
5713 if (match(Op0, m_OneUse(m_c_And(m_Neg(m_Specific(Op1)), m_Specific(Op1)))))
5714 A = Op1;
5715 else if (match(Op1,
5717 A = Op0;
5718
5719 CheckIs = Pred == ICmpInst::ICMP_EQ;
5720 } else if (ICmpInst::isUnsigned(Pred)) {
5721 // (A ^ (A-1)) u>= A --> ctpop(A) < 2 (two commuted variants)
5722 // ((A-1) ^ A) u< A --> ctpop(A) > 1 (two commuted variants)
5723
5724 if ((Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
5726 m_Specific(Op1))))) {
5727 A = Op1;
5728 CheckIs = Pred == ICmpInst::ICMP_UGE;
5729 } else if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) &&
5731 m_Specific(Op0))))) {
5732 A = Op0;
5733 CheckIs = Pred == ICmpInst::ICMP_ULE;
5734 }
5735 }
5736
5737 if (A) {
5738 Type *Ty = A->getType();
5739 CallInst *CtPop = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, A);
5740 return CheckIs ? new ICmpInst(ICmpInst::ICMP_ULT, CtPop,
5741 ConstantInt::get(Ty, 2))
5742 : new ICmpInst(ICmpInst::ICMP_UGT, CtPop,
5743 ConstantInt::get(Ty, 1));
5744 }
5745
5746 return nullptr;
5747}
5748
5750 if (!I.isEquality())
5751 return nullptr;
5752
5753 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
5754 const CmpInst::Predicate Pred = I.getPredicate();
5755 Value *A, *B, *C, *D;
5756 if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) {
5757 if (A == Op1 || B == Op1) { // (A^B) == A -> B == 0
5758 Value *OtherVal = A == Op1 ? B : A;
5759 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType()));
5760 }
5761
5762 if (match(Op1, m_Xor(m_Value(C), m_Value(D)))) {
5763 // A^c1 == C^c2 --> A == C^(c1^c2)
5764 ConstantInt *C1, *C2;
5765 if (match(B, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2)) &&
5766 Op1->hasOneUse()) {
5767 Constant *NC = Builder.getInt(C1->getValue() ^ C2->getValue());
5769 return new ICmpInst(Pred, A, Xor);
5770 }
5771
5772 // A^B == A^D -> B == D
5773 if (A == C)
5774 return new ICmpInst(Pred, B, D);
5775 if (A == D)
5776 return new ICmpInst(Pred, B, C);
5777 if (B == C)
5778 return new ICmpInst(Pred, A, D);
5779 if (B == D)
5780 return new ICmpInst(Pred, A, C);
5781 }
5782 }
5783
5784 if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && (A == Op0 || B == Op0)) {
5785 // A == (A^B) -> B == 0
5786 Value *OtherVal = A == Op0 ? B : A;
5787 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType()));
5788 }
5789
5790 // (X&Z) == (Y&Z) -> (X^Y) & Z == 0
5791 if (match(Op0, m_And(m_Value(A), m_Value(B))) &&
5792 match(Op1, m_And(m_Value(C), m_Value(D)))) {
5793 Value *X = nullptr, *Y = nullptr, *Z = nullptr;
5794
5795 if (A == C) {
5796 X = B;
5797 Y = D;
5798 Z = A;
5799 } else if (A == D) {
5800 X = B;
5801 Y = C;
5802 Z = A;
5803 } else if (B == C) {
5804 X = A;
5805 Y = D;
5806 Z = B;
5807 } else if (B == D) {
5808 X = A;
5809 Y = C;
5810 Z = B;
5811 }
5812
5813 if (X) {
5814 // If X^Y is a negative power of two, then `icmp eq/ne (Z & NegP2), 0`
5815 // will fold to `icmp ult/uge Z, -NegP2` incurringb no additional
5816 // instructions.
5817 const APInt *C0, *C1;
5818 bool XorIsNegP2 = match(X, m_APInt(C0)) && match(Y, m_APInt(C1)) &&
5819 (*C0 ^ *C1).isNegatedPowerOf2();
5820
5821 // If either Op0/Op1 are both one use or X^Y will constant fold and one of
5822 // Op0/Op1 are one use, proceed. In those cases we are instruction neutral
5823 // but `icmp eq/ne A, 0` is easier to analyze than `icmp eq/ne A, B`.
5824 int UseCnt =
5825 int(Op0->hasOneUse()) + int(Op1->hasOneUse()) +
5826 (int(match(X, m_ImmConstant()) && match(Y, m_ImmConstant())));
5827 if (XorIsNegP2 || UseCnt >= 2) {
5828 // Build (X^Y) & Z
5829 Op1 = Builder.CreateXor(X, Y);
5830 Op1 = Builder.CreateAnd(Op1, Z);
5831 return new ICmpInst(Pred, Op1, Constant::getNullValue(Op1->getType()));
5832 }
5833 }
5834 }
5835
5836 {
5837 // Similar to above, but specialized for constant because invert is needed:
5838 // (X | C) == (Y | C) --> (X ^ Y) & ~C == 0
5839 Value *X, *Y;
5840 Constant *C;
5841 if (match(Op0, m_OneUse(m_Or(m_Value(X), m_Constant(C)))) &&
5842 match(Op1, m_OneUse(m_Or(m_Value(Y), m_Specific(C))))) {
5845 return new ICmpInst(Pred, And, Constant::getNullValue(And->getType()));
5846 }
5847 }
5848
5849 if (match(Op1, m_ZExt(m_Value(A))) &&
5850 (Op0->hasOneUse() || Op1->hasOneUse())) {
5851 // (B & (Pow2C-1)) == zext A --> A == trunc B
5852 // (B & (Pow2C-1)) != zext A --> A != trunc B
5853 const APInt *MaskC;
5854 if (match(Op0, m_And(m_Value(B), m_LowBitMask(MaskC))) &&
5855 MaskC->countr_one() == A->getType()->getScalarSizeInBits())
5856 return new ICmpInst(Pred, A, Builder.CreateTrunc(B, A->getType()));
5857 }
5858
5859 // (A >> C) == (B >> C) --> (A^B) u< (1 << C)
5860 // For lshr and ashr pairs.
5861 const APInt *AP1, *AP2;
5862 if ((match(Op0, m_OneUse(m_LShr(m_Value(A), m_APIntAllowPoison(AP1)))) &&
5863 match(Op1, m_OneUse(m_LShr(m_Value(B), m_APIntAllowPoison(AP2))))) ||
5864 (match(Op0, m_OneUse(m_AShr(m_Value(A), m_APIntAllowPoison(AP1)))) &&
5865 match(Op1, m_OneUse(m_AShr(m_Value(B), m_APIntAllowPoison(AP2)))))) {
5866 if (AP1 != AP2)
5867 return nullptr;
5868 unsigned TypeBits = AP1->getBitWidth();
5869 unsigned ShAmt = AP1->getLimitedValue(TypeBits);
5870 if (ShAmt < TypeBits && ShAmt != 0) {
5871 ICmpInst::Predicate NewPred =
5873 Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted");
5874 APInt CmpVal = APInt::getOneBitSet(TypeBits, ShAmt);
5875 return new ICmpInst(NewPred, Xor, ConstantInt::get(A->getType(), CmpVal));
5876 }
5877 }
5878
5879 // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0
5880 ConstantInt *Cst1;
5881 if (match(Op0, m_OneUse(m_Shl(m_Value(A), m_ConstantInt(Cst1)))) &&
5882 match(Op1, m_OneUse(m_Shl(m_Value(B), m_Specific(Cst1))))) {
5883 unsigned TypeBits = Cst1->getBitWidth();
5884 unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits);
5885 if (ShAmt < TypeBits && ShAmt != 0) {
5886 Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted");
5887 APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt);
5889 I.getName() + ".mask");
5890 return new ICmpInst(Pred, And, Constant::getNullValue(Cst1->getType()));
5891 }
5892 }
5893
5894 // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to
5895 // "icmp (and X, mask), cst"
5896 uint64_t ShAmt = 0;
5897 if (Op0->hasOneUse() &&
5898 match(Op0, m_Trunc(m_OneUse(m_LShr(m_Value(A), m_ConstantInt(ShAmt))))) &&
5899 match(Op1, m_ConstantInt(Cst1)) &&
5900 // Only do this when A has multiple uses. This is most important to do
5901 // when it exposes other optimizations.
5902 !A->hasOneUse()) {
5903 unsigned ASize = cast<IntegerType>(A->getType())->getPrimitiveSizeInBits();
5904
5905 if (ShAmt < ASize) {
5906 APInt MaskV =
5908 MaskV <<= ShAmt;
5909
5910 APInt CmpV = Cst1->getValue().zext(ASize);
5911 CmpV <<= ShAmt;
5912
5913 Value *Mask = Builder.CreateAnd(A, Builder.getInt(MaskV));
5914 return new ICmpInst(Pred, Mask, Builder.getInt(CmpV));
5915 }
5916 }
5917
5919 return ICmp;
5920
5921 // Match icmp eq (trunc (lshr A, BW), (ashr (trunc A), BW-1)), which checks the
5922 // top BW/2 + 1 bits are all the same. Create "A >=s INT_MIN && A <=s INT_MAX",
5923 // which we generate as "icmp ult (add A, 2^(BW-1)), 2^BW" to skip a few steps
5924 // of instcombine.
5925 unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
5926 if (match(Op0, m_AShr(m_Trunc(m_Value(A)), m_SpecificInt(BitWidth - 1))) &&
5928 A->getType()->getScalarSizeInBits() == BitWidth * 2 &&
5929 (I.getOperand(0)->hasOneUse() || I.getOperand(1)->hasOneUse())) {
5931 Value *Add = Builder.CreateAdd(A, ConstantInt::get(A->getType(), C));
5932 return new ICmpInst(Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT
5934 Add, ConstantInt::get(A->getType(), C.shl(1)));
5935 }
5936
5937 // Canonicalize:
5938 // Assume B_Pow2 != 0
5939 // 1. A & B_Pow2 != B_Pow2 -> A & B_Pow2 == 0
5940 // 2. A & B_Pow2 == B_Pow2 -> A & B_Pow2 != 0
5941 if (match(Op0, m_c_And(m_Specific(Op1), m_Value())) &&
5942 isKnownToBeAPowerOfTwo(Op1, /* OrZero */ false, 0, &I))
5943 return new ICmpInst(CmpInst::getInversePredicate(Pred), Op0,
5945
5946 if (match(Op1, m_c_And(m_Specific(Op0), m_Value())) &&
5947 isKnownToBeAPowerOfTwo(Op0, /* OrZero */ false, 0, &I))
5948 return new ICmpInst(CmpInst::getInversePredicate(Pred), Op1,
5949 ConstantInt::getNullValue(Op1->getType()));
5950
5951 // Canonicalize:
5952 // icmp eq/ne X, OneUse(rotate-right(X))
5953 // -> icmp eq/ne X, rotate-left(X)
5954 // We generally try to convert rotate-right -> rotate-left, this just
5955 // canonicalizes another case.
5956 if (match(&I, m_c_ICmp(m_Value(A),
5957 m_OneUse(m_Intrinsic<Intrinsic::fshr>(
5958 m_Deferred(A), m_Deferred(A), m_Value(B))))))
5959 return new ICmpInst(
5960 Pred, A,
5961 Builder.CreateIntrinsic(Op0->getType(), Intrinsic::fshl, {A, A, B}));
5962
5963 // Canonicalize:
5964 // icmp eq/ne OneUse(A ^ Cst), B --> icmp eq/ne (A ^ B), Cst
5965 Constant *Cst;
5968 return new ICmpInst(Pred, Builder.CreateXor(A, B), Cst);
5969
5970 {
5971 // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2)
5972 auto m_Matcher =
5975 m_Sub(m_Value(B), m_Deferred(A)));
5976 std::optional<bool> IsZero = std::nullopt;
5977 if (match(&I, m_c_ICmp(m_OneUse(m_c_And(m_Value(A), m_Matcher)),
5978 m_Deferred(A))))
5979 IsZero = false;
5980 // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0)
5981 else if (match(&I,
5982 m_ICmp(m_OneUse(m_c_And(m_Value(A), m_Matcher)), m_Zero())))
5983 IsZero = true;
5984
5985 if (IsZero && isKnownToBeAPowerOfTwo(A, /* OrZero */ true, /*Depth*/ 0, &I))
5986 // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2)
5987 // -> (icmp eq/ne (and X, P2), 0)
5988 // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0)
5989 // -> (icmp eq/ne (and X, P2), P2)
5990 return new ICmpInst(Pred, Builder.CreateAnd(B, A),
5991 *IsZero ? A
5992 : ConstantInt::getNullValue(A->getType()));
5993 }
5994
5995 return nullptr;
5996}
5997
5999 ICmpInst::Predicate Pred = ICmp.getPredicate();
6000 Value *Op0 = ICmp.getOperand(0), *Op1 = ICmp.getOperand(1);
6001
6002 // Try to canonicalize trunc + compare-to-constant into a mask + cmp.
6003 // The trunc masks high bits while the compare may effectively mask low bits.
6004 Value *X;
6005 const APInt *C;
6006 if (!match(Op0, m_OneUse(m_Trunc(m_Value(X)))) || !match(Op1, m_APInt(C)))
6007 return nullptr;
6008
6009 // This matches patterns corresponding to tests of the signbit as well as:
6010 // (trunc X) pred C2 --> (X & Mask) == C
6011 if (auto Res = decomposeBitTestICmp(Op0, Op1, Pred, /*WithTrunc=*/true,
6012 /*AllowNonZeroC=*/true)) {
6013 Value *And = Builder.CreateAnd(Res->X, Res->Mask);
6014 Constant *C = ConstantInt::get(Res->X->getType(), Res->C);
6015 return new ICmpInst(Res->Pred, And, C);
6016 }
6017
6018 unsigned SrcBits = X->getType()->getScalarSizeInBits();
6019 if (auto *II = dyn_cast<IntrinsicInst>(X)) {
6020 if (II->getIntrinsicID() == Intrinsic::cttz ||
6021 II->getIntrinsicID() == Intrinsic::ctlz) {
6022 unsigned MaxRet = SrcBits;
6023 // If the "is_zero_poison" argument is set, then we know at least
6024 // one bit is set in the input, so the result is always at least one
6025 // less than the full bitwidth of that input.
6026 if (match(II->getArgOperand(1), m_One()))
6027 MaxRet--;
6028
6029 // Make sure the destination is wide enough to hold the largest output of
6030 // the intrinsic.
6031 if (llvm::Log2_32(MaxRet) + 1 <= Op0->getType()->getScalarSizeInBits())
6032 if (Instruction *I =
6033 foldICmpIntrinsicWithConstant(ICmp, II, C->zext(SrcBits)))
6034 return I;
6035 }
6036 }
6037
6038 return nullptr;
6039}
6040
6042 assert(isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0");
6043 auto *CastOp0 = cast<CastInst>(ICmp.getOperand(0));
6044 Value *X;
6045 if (!match(CastOp0, m_ZExtOrSExt(m_Value(X))))
6046 return nullptr;
6047
6048 bool IsSignedExt = CastOp0->getOpcode() == Instruction::SExt;
6049 bool IsSignedCmp = ICmp.isSigned();
6050
6051 // icmp Pred (ext X), (ext Y)
6052 Value *Y;
6053 if (match(ICmp.getOperand(1), m_ZExtOrSExt(m_Value(Y)))) {
6054 bool IsZext0 = isa<ZExtInst>(ICmp.getOperand(0));
6055 bool IsZext1 = isa<ZExtInst>(ICmp.getOperand(1));
6056
6057 if (IsZext0 != IsZext1) {
6058 // If X and Y and both i1
6059 // (icmp eq/ne (zext X) (sext Y))
6060 // eq -> (icmp eq (or X, Y), 0)
6061 // ne -> (icmp ne (or X, Y), 0)
6062 if (ICmp.isEquality() && X->getType()->isIntOrIntVectorTy(1) &&
6063 Y->getType()->isIntOrIntVectorTy(1))
6064 return new ICmpInst(ICmp.getPredicate(), Builder.CreateOr(X, Y),
6065 Constant::getNullValue(X->getType()));
6066
6067 // If we have mismatched casts and zext has the nneg flag, we can
6068 // treat the "zext nneg" as "sext". Otherwise, we cannot fold and quit.
6069
6070 auto *NonNegInst0 = dyn_cast<PossiblyNonNegInst>(ICmp.getOperand(0));
6071 auto *NonNegInst1 = dyn_cast<PossiblyNonNegInst>(ICmp.getOperand(1));
6072
6073 bool IsNonNeg0 = NonNegInst0 && NonNegInst0->hasNonNeg();
6074 bool IsNonNeg1 = NonNegInst1 && NonNegInst1->hasNonNeg();
6075
6076 if ((IsZext0 && IsNonNeg0) || (IsZext1 && IsNonNeg1))
6077 IsSignedExt = true;
6078 else
6079 return nullptr;
6080 }
6081
6082 // Not an extension from the same type?
6083 Type *XTy = X->getType(), *YTy = Y->getType();
6084 if (XTy != YTy) {
6085 // One of the casts must have one use because we are creating a new cast.
6086 if (!ICmp.getOperand(0)->hasOneUse() && !ICmp.getOperand(1)->hasOneUse())
6087 return nullptr;
6088 // Extend the narrower operand to the type of the wider operand.
6089 CastInst::CastOps CastOpcode =
6090 IsSignedExt ? Instruction::SExt : Instruction::ZExt;
6091 if (XTy->getScalarSizeInBits() < YTy->getScalarSizeInBits())
6092 X = Builder.CreateCast(CastOpcode, X, YTy);
6093 else if (YTy->getScalarSizeInBits() < XTy->getScalarSizeInBits())
6094 Y = Builder.CreateCast(CastOpcode, Y, XTy);
6095 else
6096 return nullptr;
6097 }
6098
6099 // (zext X) == (zext Y) --> X == Y
6100 // (sext X) == (sext Y) --> X == Y
6101 if (ICmp.isEquality())
6102 return new ICmpInst(ICmp.getPredicate(), X, Y);
6103
6104 // A signed comparison of sign extended values simplifies into a
6105 // signed comparison.
6106 if (IsSignedCmp && IsSignedExt)
6107 return new ICmpInst(ICmp.getPredicate(), X, Y);
6108
6109 // The other three cases all fold into an unsigned comparison.
6110 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Y);
6111 }
6112
6113 // Below here, we are only folding a compare with constant.
6114 auto *C = dyn_cast<Constant>(ICmp.getOperand(1));
6115 if (!C)
6116 return nullptr;
6117
6118 // If a lossless truncate is possible...
6119 Type *SrcTy = CastOp0->getSrcTy();
6120 Constant *Res = getLosslessTrunc(C, SrcTy, CastOp0->getOpcode());
6121 if (Res) {
6122 if (ICmp.isEquality())
6123 return new ICmpInst(ICmp.getPredicate(), X, Res);
6124
6125 // A signed comparison of sign extended values simplifies into a
6126 // signed comparison.
6127 if (IsSignedExt && IsSignedCmp)
6128 return new ICmpInst(ICmp.getPredicate(), X, Res);
6129
6130 // The other three cases all fold into an unsigned comparison.
6131 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Res);
6132 }
6133
6134 // The re-extended constant changed, partly changed (in the case of a vector),
6135 // or could not be determined to be equal (in the case of a constant
6136 // expression), so the constant cannot be represented in the shorter type.
6137 // All the cases that fold to true or false will have already been handled
6138 // by simplifyICmpInst, so only deal with the tricky case.
6139 if (IsSignedCmp || !IsSignedExt || !isa<ConstantInt>(C))
6140 return nullptr;
6141
6142 // Is source op positive?
6143 // icmp ult (sext X), C --> icmp sgt X, -1
6144 if (ICmp.getPredicate() == ICmpInst::ICMP_ULT)
6146
6147 // Is source op negative?
6148 // icmp ugt (sext X), C --> icmp slt X, 0
6149 assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!");
6151}
6152
6153/// Handle icmp (cast x), (cast or constant).
6155 // If any operand of ICmp is a inttoptr roundtrip cast then remove it as
6156 // icmp compares only pointer's value.
6157 // icmp (inttoptr (ptrtoint p1)), p2 --> icmp p1, p2.
6158 Value *SimplifiedOp0 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(0));
6159 Value *SimplifiedOp1 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(1));
6160 if (SimplifiedOp0 || SimplifiedOp1)
6161 return new ICmpInst(ICmp.getPredicate(),
6162 SimplifiedOp0 ? SimplifiedOp0 : ICmp.getOperand(0),
6163 SimplifiedOp1 ? SimplifiedOp1 : ICmp.getOperand(1));
6164
6165 auto *CastOp0 = dyn_cast<CastInst>(ICmp.getOperand(0));
6166 if (!CastOp0)
6167 return nullptr;
6168 if (!isa<Constant>(ICmp.getOperand(1)) && !isa<CastInst>(ICmp.getOperand(1)))
6169 return nullptr;
6170
6171 Value *Op0Src = CastOp0->getOperand(0);
6172 Type *SrcTy = CastOp0->getSrcTy();
6173 Type *DestTy = CastOp0->getDestTy();
6174
6175 // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the
6176 // integer type is the same size as the pointer type.
6177 auto CompatibleSizes = [&](Type *PtrTy, Type *IntTy) {
6178 if (isa<VectorType>(PtrTy)) {
6179 PtrTy = cast<VectorType>(PtrTy)->getElementType();
6180 IntTy = cast<VectorType>(IntTy)->getElementType();
6181 }
6182 return DL.getPointerTypeSizeInBits(PtrTy) == IntTy->getIntegerBitWidth();
6183 };
6184 if (CastOp0->getOpcode() == Instruction::PtrToInt &&
6185 CompatibleSizes(SrcTy, DestTy)) {
6186 Value *NewOp1 = nullptr;
6187 if (auto *PtrToIntOp1 = dyn_cast<PtrToIntOperator>(ICmp.getOperand(1))) {
6188 Value *PtrSrc = PtrToIntOp1->getOperand(0);
6189 if (PtrSrc->getType() == Op0Src->getType())
6190 NewOp1 = PtrToIntOp1->getOperand(0);
6191 } else if (auto *RHSC = dyn_cast<Constant>(ICmp.getOperand(1))) {
6192 NewOp1 = ConstantExpr::getIntToPtr(RHSC, SrcTy);
6193 }
6194
6195 if (NewOp1)
6196 return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1);
6197 }
6198
6199 // Do the same in the other direction for icmp (inttoptr x), (inttoptr/c).
6200 if (CastOp0->getOpcode() == Instruction::IntToPtr &&
6201 CompatibleSizes(DestTy, SrcTy)) {
6202 Value *NewOp1 = nullptr;
6203 if (auto *IntToPtrOp1 = dyn_cast<IntToPtrInst>(ICmp.getOperand(1))) {
6204 Value *IntSrc = IntToPtrOp1->getOperand(0);
6205 if (IntSrc->getType() == Op0Src->getType())
6206 NewOp1 = IntToPtrOp1->getOperand(0);
6207 } else if (auto *RHSC = dyn_cast<Constant>(ICmp.getOperand(1))) {
6208 NewOp1 = ConstantFoldConstant(ConstantExpr::getPtrToInt(RHSC, SrcTy), DL);
6209 }
6210
6211 if (NewOp1)
6212 return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1);
6213 }
6214
6215 if (Instruction *R = foldICmpWithTrunc(ICmp))
6216 return R;
6217
6218 return foldICmpWithZextOrSext(ICmp);
6219}
6220
6221static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS, bool IsSigned) {
6222 switch (BinaryOp) {
6223 default:
6224 llvm_unreachable("Unsupported binary op");
6225 case Instruction::Add:
6226 case Instruction::Sub:
6227 return match(RHS, m_Zero());
6228 case Instruction::Mul:
6229 return !(RHS->getType()->isIntOrIntVectorTy(1) && IsSigned) &&
6230 match(RHS, m_One());
6231 }
6232}
6233
6236 bool IsSigned, Value *LHS, Value *RHS,
6237 Instruction *CxtI) const {
6238 switch (BinaryOp) {
6239 default:
6240 llvm_unreachable("Unsupported binary op");
6241 case Instruction::Add:
6242 if (IsSigned)
6243 return computeOverflowForSignedAdd(LHS, RHS, CxtI);
6244 else
6245 return computeOverflowForUnsignedAdd(LHS, RHS, CxtI);
6246 case Instruction::Sub:
6247 if (IsSigned)
6248 return computeOverflowForSignedSub(LHS, RHS, CxtI);
6249 else
6250 return computeOverflowForUnsignedSub(LHS, RHS, CxtI);
6251 case Instruction::Mul:
6252 if (IsSigned)
6253 return computeOverflowForSignedMul(LHS, RHS, CxtI);
6254 else
6255 return computeOverflowForUnsignedMul(LHS, RHS, CxtI);
6256 }
6257}
6258
6259bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp,
6260 bool IsSigned, Value *LHS,
6261 Value *RHS, Instruction &OrigI,
6262 Value *&Result,
6263 Constant *&Overflow) {
6264 if (OrigI.isCommutative() && isa<Constant>(LHS) && !isa<Constant>(RHS))
6265 std::swap(LHS, RHS);
6266
6267 // If the overflow check was an add followed by a compare, the insertion point
6268 // may be pointing to the compare. We want to insert the new instructions
6269 // before the add in case there are uses of the add between the add and the
6270 // compare.
6271 Builder.SetInsertPoint(&OrigI);
6272
6273 Type *OverflowTy = Type::getInt1Ty(LHS->getContext());
6274 if (auto *LHSTy = dyn_cast<VectorType>(LHS->getType()))
6275 OverflowTy = VectorType::get(OverflowTy, LHSTy->getElementCount());
6276
6277 if (isNeutralValue(BinaryOp, RHS, IsSigned)) {
6278 Result = LHS;
6279 Overflow = ConstantInt::getFalse(OverflowTy);
6280 return true;
6281 }
6282
6283 switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, &OrigI)) {
6285 return false;
6288 Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
6289 Result->takeName(&OrigI);
6290 Overflow = ConstantInt::getTrue(OverflowTy);
6291 return true;
6293 Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
6294 Result->takeName(&OrigI);
6295 Overflow = ConstantInt::getFalse(OverflowTy);
6296 if (auto *Inst = dyn_cast<Instruction>(Result)) {
6297 if (IsSigned)
6298 Inst->setHasNoSignedWrap();
6299 else
6300 Inst->setHasNoUnsignedWrap();
6301 }
6302 return true;
6303 }
6304
6305 llvm_unreachable("Unexpected overflow result");
6306}
6307
6308/// Recognize and process idiom involving test for multiplication
6309/// overflow.
6310///
6311/// The caller has matched a pattern of the form:
6312/// I = cmp u (mul(zext A, zext B), V
6313/// The function checks if this is a test for overflow and if so replaces
6314/// multiplication with call to 'mul.with.overflow' intrinsic.
6315///
6316/// \param I Compare instruction.
6317/// \param MulVal Result of 'mult' instruction. It is one of the arguments of
6318/// the compare instruction. Must be of integer type.
6319/// \param OtherVal The other argument of compare instruction.
6320/// \returns Instruction which must replace the compare instruction, NULL if no
6321/// replacement required.
6323 const APInt *OtherVal,
6324 InstCombinerImpl &IC) {
6325 // Don't bother doing this transformation for pointers, don't do it for
6326 // vectors.
6327 if (!isa<IntegerType>(MulVal->getType()))
6328 return nullptr;
6329
6330 auto *MulInstr = dyn_cast<Instruction>(MulVal);
6331 if (!MulInstr)
6332 return nullptr;
6333 assert(MulInstr->getOpcode() == Instruction::Mul);
6334
6335 auto *LHS = cast<ZExtInst>(MulInstr->getOperand(0)),
6336 *RHS = cast<ZExtInst>(MulInstr->getOperand(1));
6337 assert(LHS->getOpcode() == Instruction::ZExt);
6338 assert(RHS->getOpcode() == Instruction::ZExt);
6339 Value *A = LHS->getOperand(0), *B = RHS->getOperand(0);
6340
6341 // Calculate type and width of the result produced by mul.with.overflow.
6342 Type *TyA = A->getType(), *TyB = B->getType();
6343 unsigned WidthA = TyA->getPrimitiveSizeInBits(),
6344 WidthB = TyB->getPrimitiveSizeInBits();
6345 unsigned MulWidth;
6346 Type *MulType;
6347 if (WidthB > WidthA) {
6348 MulWidth = WidthB;
6349 MulType = TyB;
6350 } else {
6351 MulWidth = WidthA;
6352 MulType = TyA;
6353 }
6354
6355 // In order to replace the original mul with a narrower mul.with.overflow,
6356 // all uses must ignore upper bits of the product. The number of used low
6357 // bits must be not greater than the width of mul.with.overflow.
6358 if (MulVal->hasNUsesOrMore(2))
6359 for (User *U : MulVal->users()) {
6360 if (U == &I)
6361 continue;
6362 if (TruncInst *TI = dyn_cast<TruncInst>(U)) {
6363 // Check if truncation ignores bits above MulWidth.
6364 unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits();
6365 if (TruncWidth > MulWidth)
6366 return nullptr;
6367 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) {
6368 // Check if AND ignores bits above MulWidth.
6369 if (BO->getOpcode() != Instruction::And)
6370 return nullptr;
6371 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->getOperand(1))) {
6372 const APInt &CVal = CI->getValue();
6373 if (CVal.getBitWidth() - CVal.countl_zero() > MulWidth)
6374 return nullptr;
6375 } else {
6376 // In this case we could have the operand of the binary operation
6377 // being defined in another block, and performing the replacement
6378 // could break the dominance relation.
6379 return nullptr;
6380 }
6381 } else {
6382 // Other uses prohibit this transformation.
6383 return nullptr;
6384 }
6385 }
6386
6387 // Recognize patterns
6388 switch (I.getPredicate()) {
6389 case ICmpInst::ICMP_UGT: {
6390 // Recognize pattern:
6391 // mulval = mul(zext A, zext B)
6392 // cmp ugt mulval, max
6393 APInt MaxVal = APInt::getMaxValue(MulWidth);
6394 MaxVal = MaxVal.zext(OtherVal->getBitWidth());
6395 if (MaxVal.eq(*OtherVal))
6396 break; // Recognized
6397 return nullptr;
6398 }
6399
6400 case ICmpInst::ICMP_ULT: {
6401 // Recognize pattern:
6402 // mulval = mul(zext A, zext B)
6403 // cmp ule mulval, max + 1
6404 APInt MaxVal = APInt::getOneBitSet(OtherVal->getBitWidth(), MulWidth);
6405 if (MaxVal.eq(*OtherVal))
6406 break; // Recognized
6407 return nullptr;
6408 }
6409
6410 default:
6411 return nullptr;
6412 }
6413
6414 InstCombiner::BuilderTy &Builder = IC.Builder;
6415 Builder.SetInsertPoint(MulInstr);
6416
6417 // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B)
6418 Value *MulA = A, *MulB = B;
6419 if (WidthA < MulWidth)
6420 MulA = Builder.CreateZExt(A, MulType);
6421 if (WidthB < MulWidth)
6422 MulB = Builder.CreateZExt(B, MulType);
6423 CallInst *Call =
6424 Builder.CreateIntrinsic(Intrinsic::umul_with_overflow, MulType,
6425 {MulA, MulB}, /*FMFSource=*/nullptr, "umul");
6426 IC.addToWorklist(MulInstr);
6427
6428 // If there are uses of mul result other than the comparison, we know that
6429 // they are truncation or binary AND. Change them to use result of
6430 // mul.with.overflow and adjust properly mask/size.
6431 if (MulVal->hasNUsesOrMore(2)) {
6432 Value *Mul = Builder.CreateExtractValue(Call, 0, "umul.value");
6433 for (User *U : make_early_inc_range(MulVal->users())) {
6434 if (U == &I)
6435 continue;
6436 if (TruncInst *TI = dyn_cast<TruncInst>(U)) {
6437 if (TI->getType()->getPrimitiveSizeInBits() == MulWidth)
6438 IC.replaceInstUsesWith(*TI, Mul);
6439 else
6440 TI->setOperand(0, Mul);
6441 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) {
6442 assert(BO->getOpcode() == Instruction::And);
6443 // Replace (mul & mask) --> zext (mul.with.overflow & short_mask)
6444 ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1));
6445 APInt ShortMask = CI->getValue().trunc(MulWidth);
6446 Value *ShortAnd = Builder.CreateAnd(Mul, ShortMask);
6447 Value *Zext = Builder.CreateZExt(ShortAnd, BO->getType());
6448 IC.replaceInstUsesWith(*BO, Zext);
6449 } else {
6450 llvm_unreachable("Unexpected Binary operation");
6451 }
6452 IC.addToWorklist(cast<Instruction>(U));
6453 }
6454 }
6455
6456 // The original icmp gets replaced with the overflow value, maybe inverted
6457 // depending on predicate.
6458 if (I.getPredicate() == ICmpInst::ICMP_ULT) {
6459 Value *Res = Builder.CreateExtractValue(Call, 1);
6460 return BinaryOperator::CreateNot(Res);
6461 }
6462
6463 return ExtractValueInst::Create(Call, 1);
6464}
6465
6466/// When performing a comparison against a constant, it is possible that not all
6467/// the bits in the LHS are demanded. This helper method computes the mask that
6468/// IS demanded.
6470 const APInt *RHS;
6471 if (!match(I.getOperand(1), m_APInt(RHS)))
6473
6474 // If this is a normal comparison, it demands all bits. If it is a sign bit
6475 // comparison, it only demands the sign bit.
6476 bool UnusedBit;
6477 if (isSignBitCheck(I.getPredicate(), *RHS, UnusedBit))
6479
6480 switch (I.getPredicate()) {
6481 // For a UGT comparison, we don't care about any bits that
6482 // correspond to the trailing ones of the comparand. The value of these
6483 // bits doesn't impact the outcome of the comparison, because any value
6484 // greater than the RHS must differ in a bit higher than these due to carry.
6485 case ICmpInst::ICMP_UGT:
6486 return APInt::getBitsSetFrom(BitWidth, RHS->countr_one());
6487
6488 // Similarly, for a ULT comparison, we don't care about the trailing zeros.
6489 // Any value less than the RHS must differ in a higher bit because of carries.
6490 case ICmpInst::ICMP_ULT:
6491 return APInt::getBitsSetFrom(BitWidth, RHS->countr_zero());
6492
6493 default:
6495 }
6496}
6497
6498/// Check that one use is in the same block as the definition and all
6499/// other uses are in blocks dominated by a given block.
6500///
6501/// \param DI Definition
6502/// \param UI Use
6503/// \param DB Block that must dominate all uses of \p DI outside
6504/// the parent block
6505/// \return true when \p UI is the only use of \p DI in the parent block
6506/// and all other uses of \p DI are in blocks dominated by \p DB.
6507///
6509 const Instruction *UI,
6510 const BasicBlock *DB) const {
6511 assert(DI && UI && "Instruction not defined\n");
6512 // Ignore incomplete definitions.
6513 if (!DI->getParent())
6514 return false;
6515 // DI and UI must be in the same block.
6516 if (DI->getParent() != UI->getParent())
6517 return false;
6518 // Protect from self-referencing blocks.
6519 if (DI->getParent() == DB)
6520 return false;
6521 for (const User *U : DI->users()) {
6522 auto *Usr = cast<Instruction>(U);
6523 if (Usr != UI && !DT.dominates(DB, Usr->getParent()))
6524 return false;
6525 }
6526 return true;
6527}
6528
6529/// Return true when the instruction sequence within a block is select-cmp-br.
6530static bool isChainSelectCmpBranch(const SelectInst *SI) {
6531 const BasicBlock *BB = SI->getParent();
6532 if (!BB)
6533 return false;
6534 auto *BI = dyn_cast_or_null<BranchInst>(BB->getTerminator());
6535 if (!BI || BI->getNumSuccessors() != 2)
6536 return false;
6537 auto *IC = dyn_cast<ICmpInst>(BI->getCondition());
6538 if (!IC || (IC->getOperand(0) != SI && IC->getOperand(1) != SI))
6539 return false;
6540 return true;
6541}
6542
6543/// True when a select result is replaced by one of its operands
6544/// in select-icmp sequence. This will eventually result in the elimination
6545/// of the select.
6546///
6547/// \param SI Select instruction
6548/// \param Icmp Compare instruction
6549/// \param SIOpd Operand that replaces the select
6550///
6551/// Notes:
6552/// - The replacement is global and requires dominator information
6553/// - The caller is responsible for the actual replacement
6554///
6555/// Example:
6556///
6557/// entry:
6558/// %4 = select i1 %3, %C* %0, %C* null
6559/// %5 = icmp eq %C* %4, null
6560/// br i1 %5, label %9, label %7
6561/// ...
6562/// ; <label>:7 ; preds = %entry
6563/// %8 = getelementptr inbounds %C* %4, i64 0, i32 0
6564/// ...
6565///
6566/// can be transformed to
6567///
6568/// %5 = icmp eq %C* %0, null
6569/// %6 = select i1 %3, i1 %5, i1 true
6570/// br i1 %6, label %9, label %7
6571/// ...
6572/// ; <label>:7 ; preds = %entry
6573/// %8 = getelementptr inbounds %C* %0, i64 0, i32 0 // replace by %0!
6574///
6575/// Similar when the first operand of the select is a constant or/and
6576/// the compare is for not equal rather than equal.
6577///
6578/// NOTE: The function is only called when the select and compare constants
6579/// are equal, the optimization can work only for EQ predicates. This is not a
6580/// major restriction since a NE compare should be 'normalized' to an equal
6581/// compare, which usually happens in the combiner and test case
6582/// select-cmp-br.ll checks for it.
6584 const ICmpInst *Icmp,
6585 const unsigned SIOpd) {
6586 assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!");
6588 BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(1);
6589 // The check for the single predecessor is not the best that can be
6590 // done. But it protects efficiently against cases like when SI's
6591 // home block has two successors, Succ and Succ1, and Succ1 predecessor
6592 // of Succ. Then SI can't be replaced by SIOpd because the use that gets
6593 // replaced can be reached on either path. So the uniqueness check
6594 // guarantees that the path all uses of SI (outside SI's parent) are on
6595 // is disjoint from all other paths out of SI. But that information
6596 // is more expensive to compute, and the trade-off here is in favor
6597 // of compile-time. It should also be noticed that we check for a single
6598 // predecessor and not only uniqueness. This to handle the situation when
6599 // Succ and Succ1 points to the same basic block.
6600 if (Succ->getSinglePredecessor() && dominatesAllUses(SI, Icmp, Succ)) {
6601 NumSel++;
6602 SI->replaceUsesOutsideBlock(SI->getOperand(SIOpd), SI->getParent());
6603 return true;
6604 }
6605 }
6606 return false;
6607}
6608
6609/// Try to fold the comparison based on range information we can get by checking
6610/// whether bits are known to be zero or one in the inputs.
6612 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
6613 Type *Ty = Op0->getType();
6614 ICmpInst::Predicate Pred = I.getPredicate();
6615
6616 // Get scalar or pointer size.
6617 unsigned BitWidth = Ty->isIntOrIntVectorTy()
6618 ? Ty->getScalarSizeInBits()
6620
6621 if (!BitWidth)
6622 return nullptr;
6623
6624 KnownBits Op0Known(BitWidth);
6625 KnownBits Op1Known(BitWidth);
6626
6627 {
6628 // Don't use dominating conditions when folding icmp using known bits. This
6629 // may convert signed into unsigned predicates in ways that other passes
6630 // (especially IndVarSimplify) may not be able to reliably undo.
6633 Op0Known, /*Depth=*/0, Q))
6634 return &I;
6635
6637 /*Depth=*/0, Q))
6638 return &I;
6639 }
6640
6641 if (!isa<Constant>(Op0) && Op0Known.isConstant())
6642 return new ICmpInst(
6643 Pred, ConstantExpr::getIntegerValue(Ty, Op0Known.getConstant()), Op1);
6644 if (!isa<Constant>(Op1) && Op1Known.isConstant())
6645 return new ICmpInst(
6646 Pred, Op0, ConstantExpr::getIntegerValue(Ty, Op1Known.getConstant()));
6647
6648 if (std::optional<bool> Res = ICmpInst::compare(Op0Known, Op1Known, Pred))
6649 return replaceInstUsesWith(I, ConstantInt::getBool(I.getType(), *Res));
6650
6651 // Given the known and unknown bits, compute a range that the LHS could be
6652 // in. Compute the Min, Max and RHS values based on the known bits. For the
6653 // EQ and NE we use unsigned values.
6654 APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0);
6655 APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0);
6656 if (I.isSigned()) {
6657 Op0Min = Op0Known.getSignedMinValue();
6658 Op0Max = Op0Known.getSignedMaxValue();
6659 Op1Min = Op1Known.getSignedMinValue();
6660 Op1Max = Op1Known.getSignedMaxValue();
6661 } else {
6662 Op0Min = Op0Known.getMinValue();
6663 Op0Max = Op0Known.getMaxValue();
6664 Op1Min = Op1Known.getMinValue();
6665 Op1Max = Op1Known.getMaxValue();
6666 }
6667
6668 // Don't break up a clamp pattern -- (min(max X, Y), Z) -- by replacing a
6669 // min/max canonical compare with some other compare. That could lead to
6670 // conflict with select canonicalization and infinite looping.
6671 // FIXME: This constraint may go away if min/max intrinsics are canonical.
6672 auto isMinMaxCmp = [&](Instruction &Cmp) {
6673 if (!Cmp.hasOneUse())
6674 return false;
6675 Value *A, *B;
6676 SelectPatternFlavor SPF = matchSelectPattern(Cmp.user_back(), A, B).Flavor;
6678 return false;
6679 return match(Op0, m_MaxOrMin(m_Value(), m_Value())) ||
6680 match(Op1, m_MaxOrMin(m_Value(), m_Value()));
6681 };
6682 if (!isMinMaxCmp(I)) {
6683 switch (Pred) {
6684 default:
6685 break;
6686 case ICmpInst::ICMP_ULT: {
6687 if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B)
6688 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6689 const APInt *CmpC;
6690 if (match(Op1, m_APInt(CmpC))) {
6691 // A <u C -> A == C-1 if min(A)+1 == C
6692 if (*CmpC == Op0Min + 1)
6693 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6694 ConstantInt::get(Op1->getType(), *CmpC - 1));
6695 // X <u C --> X == 0, if the number of zero bits in the bottom of X
6696 // exceeds the log2 of C.
6697 if (Op0Known.countMinTrailingZeros() >= CmpC->ceilLogBase2())
6698 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6699 Constant::getNullValue(Op1->getType()));
6700 }
6701 break;
6702 }
6703 case ICmpInst::ICMP_UGT: {
6704 if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B)
6705 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6706 const APInt *CmpC;
6707 if (match(Op1, m_APInt(CmpC))) {
6708 // A >u C -> A == C+1 if max(a)-1 == C
6709 if (*CmpC == Op0Max - 1)
6710 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6711 ConstantInt::get(Op1->getType(), *CmpC + 1));
6712 // X >u C --> X != 0, if the number of zero bits in the bottom of X
6713 // exceeds the log2 of C.
6714 if (Op0Known.countMinTrailingZeros() >= CmpC->getActiveBits())
6715 return new ICmpInst(ICmpInst::ICMP_NE, Op0,
6716 Constant::getNullValue(Op1->getType()));
6717 }
6718 break;
6719 }
6720 case ICmpInst::ICMP_SLT: {
6721 if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B)
6722 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6723 const APInt *CmpC;
6724 if (match(Op1, m_APInt(CmpC))) {
6725 if (*CmpC == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C
6726 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6727 ConstantInt::get(Op1->getType(), *CmpC - 1));
6728 }
6729 break;
6730 }
6731 case ICmpInst::ICMP_SGT: {
6732 if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B)
6733 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6734 const APInt *CmpC;
6735 if (match(Op1, m_APInt(CmpC))) {
6736 if (*CmpC == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C
6737 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6738 ConstantInt::get(Op1->getType(), *CmpC + 1));
6739 }
6740 break;
6741 }
6742 }
6743 }
6744
6745 // Based on the range information we know about the LHS, see if we can
6746 // simplify this comparison. For example, (x&4) < 8 is always true.
6747 switch (Pred) {
6748 default:
6749 break;
6750 case ICmpInst::ICMP_EQ:
6751 case ICmpInst::ICMP_NE: {
6752 // If all bits are known zero except for one, then we know at most one bit
6753 // is set. If the comparison is against zero, then this is a check to see if
6754 // *that* bit is set.
6755 APInt Op0KnownZeroInverted = ~Op0Known.Zero;
6756 if (Op1Known.isZero()) {
6757 // If the LHS is an AND with the same constant, look through it.
6758 Value *LHS = nullptr;
6759 const APInt *LHSC;
6760 if (!match(Op0, m_And(m_Value(LHS), m_APInt(LHSC))) ||
6761 *LHSC != Op0KnownZeroInverted)
6762 LHS = Op0;
6763
6764 Value *X;
6765 const APInt *C1;
6766 if (match(LHS, m_Shl(m_Power2(C1), m_Value(X)))) {
6767 Type *XTy = X->getType();
6768 unsigned Log2C1 = C1->countr_zero();
6769 APInt C2 = Op0KnownZeroInverted;
6770 APInt C2Pow2 = (C2 & ~(*C1 - 1)) + *C1;
6771 if (C2Pow2.isPowerOf2()) {
6772 // iff (C1 is pow2) & ((C2 & ~(C1-1)) + C1) is pow2):
6773 // ((C1 << X) & C2) == 0 -> X >= (Log2(C2+C1) - Log2(C1))
6774 // ((C1 << X) & C2) != 0 -> X < (Log2(C2+C1) - Log2(C1))
6775 unsigned Log2C2 = C2Pow2.countr_zero();
6776 auto *CmpC = ConstantInt::get(XTy, Log2C2 - Log2C1);
6777 auto NewPred =
6779 return new ICmpInst(NewPred, X, CmpC);
6780 }
6781 }
6782 }
6783
6784 // Op0 eq C_Pow2 -> Op0 ne 0 if Op0 is known to be C_Pow2 or zero.
6785 if (Op1Known.isConstant() && Op1Known.getConstant().isPowerOf2() &&
6786 (Op0Known & Op1Known) == Op0Known)
6787 return new ICmpInst(CmpInst::getInversePredicate(Pred), Op0,
6788 ConstantInt::getNullValue(Op1->getType()));
6789 break;
6790 }
6791 case ICmpInst::ICMP_SGE:
6792 if (Op1Min == Op0Max) // A >=s B -> A == B if max(A) == min(B)
6793 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6794 break;
6795 case ICmpInst::ICMP_SLE:
6796 if (Op1Max == Op0Min) // A <=s B -> A == B if min(A) == max(B)
6797 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6798 break;
6799 case ICmpInst::ICMP_UGE:
6800 if (Op1Min == Op0Max) // A >=u B -> A == B if max(A) == min(B)
6801 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6802 break;
6803 case ICmpInst::ICMP_ULE:
6804 if (Op1Max == Op0Min) // A <=u B -> A == B if min(A) == max(B)
6805 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6806 break;
6807 }
6808
6809 // Turn a signed comparison into an unsigned one if both operands are known to
6810 // have the same sign. Set samesign if possible (except for equality
6811 // predicates).
6812 if ((I.isSigned() || (I.isUnsigned() && !I.hasSameSign())) &&
6813 ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) ||
6814 (Op0Known.One.isNegative() && Op1Known.One.isNegative()))) {
6815 I.setPredicate(I.getUnsignedPredicate());
6816 I.setSameSign();
6817 return &I;
6818 }
6819
6820 return nullptr;
6821}
6822
6823/// If one operand of an icmp is effectively a bool (value range of {0,1}),
6824/// then try to reduce patterns based on that limit.
6826 Value *X, *Y;
6827 CmpPredicate Pred;
6828
6829 // X must be 0 and bool must be true for "ULT":
6830 // X <u (zext i1 Y) --> (X == 0) & Y
6831 if (match(&I, m_c_ICmp(Pred, m_Value(X), m_OneUse(m_ZExt(m_Value(Y))))) &&
6832 Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULT)
6833 return BinaryOperator::CreateAnd(Builder.CreateIsNull(X), Y);
6834
6835 // X must be 0 or bool must be true for "ULE":
6836 // X <=u (sext i1 Y) --> (X == 0) | Y
6837 if (match(&I, m_c_ICmp(Pred, m_Value(X), m_OneUse(m_SExt(m_Value(Y))))) &&
6838 Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULE)
6839 return BinaryOperator::CreateOr(Builder.CreateIsNull(X), Y);
6840
6841 // icmp eq/ne X, (zext/sext (icmp eq/ne X, C))
6842 CmpPredicate Pred1, Pred2;
6843 const APInt *C;
6844 Instruction *ExtI;
6845 if (match(&I, m_c_ICmp(Pred1, m_Value(X),
6848 m_APInt(C)))))) &&
6849 ICmpInst::isEquality(Pred1) && ICmpInst::isEquality(Pred2)) {
6850 bool IsSExt = ExtI->getOpcode() == Instruction::SExt;
6851 bool HasOneUse = ExtI->hasOneUse() && ExtI->getOperand(0)->hasOneUse();
6852 auto CreateRangeCheck = [&] {
6853 Value *CmpV1 =
6854 Builder.CreateICmp(Pred1, X, Constant::getNullValue(X->getType()));
6855 Value *CmpV2 = Builder.CreateICmp(
6856 Pred1, X, ConstantInt::getSigned(X->getType(), IsSExt ? -1 : 1));
6858 Pred1 == ICmpInst::ICMP_EQ ? Instruction::Or : Instruction::And,
6859 CmpV1, CmpV2);
6860 };
6861 if (C->isZero()) {
6862 if (Pred2 == ICmpInst::ICMP_EQ) {
6863 // icmp eq X, (zext/sext (icmp eq X, 0)) --> false
6864 // icmp ne X, (zext/sext (icmp eq X, 0)) --> true
6865 return replaceInstUsesWith(
6866 I, ConstantInt::getBool(I.getType(), Pred1 == ICmpInst::ICMP_NE));
6867 } else if (!IsSExt || HasOneUse) {
6868 // icmp eq X, (zext (icmp ne X, 0)) --> X == 0 || X == 1
6869 // icmp ne X, (zext (icmp ne X, 0)) --> X != 0 && X != 1
6870 // icmp eq X, (sext (icmp ne X, 0)) --> X == 0 || X == -1
6871 // icmp ne X, (sext (icmp ne X, 0)) --> X != 0 && X == -1
6872 return CreateRangeCheck();
6873 }
6874 } else if (IsSExt ? C->isAllOnes() : C->isOne()) {
6875 if (Pred2 == ICmpInst::ICMP_NE) {
6876 // icmp eq X, (zext (icmp ne X, 1)) --> false
6877 // icmp ne X, (zext (icmp ne X, 1)) --> true
6878 // icmp eq X, (sext (icmp ne X, -1)) --> false
6879 // icmp ne X, (sext (icmp ne X, -1)) --> true
6880 return replaceInstUsesWith(
6881 I, ConstantInt::getBool(I.getType(), Pred1 == ICmpInst::ICMP_NE));
6882 } else if (!IsSExt || HasOneUse) {
6883 // icmp eq X, (zext (icmp eq X, 1)) --> X == 0 || X == 1
6884 // icmp ne X, (zext (icmp eq X, 1)) --> X != 0 && X != 1
6885 // icmp eq X, (sext (icmp eq X, -1)) --> X == 0 || X == -1
6886 // icmp ne X, (sext (icmp eq X, -1)) --> X != 0 && X == -1
6887 return CreateRangeCheck();
6888 }
6889 } else {
6890 // when C != 0 && C != 1:
6891 // icmp eq X, (zext (icmp eq X, C)) --> icmp eq X, 0
6892 // icmp eq X, (zext (icmp ne X, C)) --> icmp eq X, 1
6893 // icmp ne X, (zext (icmp eq X, C)) --> icmp ne X, 0
6894 // icmp ne X, (zext (icmp ne X, C)) --> icmp ne X, 1
6895 // when C != 0 && C != -1:
6896 // icmp eq X, (sext (icmp eq X, C)) --> icmp eq X, 0
6897 // icmp eq X, (sext (icmp ne X, C)) --> icmp eq X, -1
6898 // icmp ne X, (sext (icmp eq X, C)) --> icmp ne X, 0
6899 // icmp ne X, (sext (icmp ne X, C)) --> icmp ne X, -1
6900 return ICmpInst::Create(
6901 Instruction::ICmp, Pred1, X,
6902 ConstantInt::getSigned(X->getType(), Pred2 == ICmpInst::ICMP_NE
6903 ? (IsSExt ? -1 : 1)
6904 : 0));
6905 }
6906 }
6907
6908 return nullptr;
6909}
6910
6911std::optional<std::pair<CmpPredicate, Constant *>>
6913 Constant *C) {
6915 "Only for relational integer predicates.");
6916
6917 Type *Type = C->getType();
6918 bool IsSigned = ICmpInst::isSigned(Pred);
6919
6921 bool WillIncrement =
6922 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT;
6923
6924 // Check if the constant operand can be safely incremented/decremented
6925 // without overflowing/underflowing.
6926 auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) {
6927 return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned);
6928 };
6929
6930 Constant *SafeReplacementConstant = nullptr;
6931 if (auto *CI = dyn_cast<ConstantInt>(C)) {
6932 // Bail out if the constant can't be safely incremented/decremented.
6933 if (!ConstantIsOk(CI))
6934 return std::nullopt;
6935 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Type)) {
6936 unsigned NumElts = FVTy->getNumElements();
6937 for (unsigned i = 0; i != NumElts; ++i) {
6938 Constant *Elt = C->getAggregateElement(i);
6939 if (!Elt)
6940 return std::nullopt;
6941
6942 if (isa<UndefValue>(Elt))
6943 continue;
6944
6945 // Bail out if we can't determine if this constant is min/max or if we
6946 // know that this constant is min/max.
6947 auto *CI = dyn_cast<ConstantInt>(Elt);
6948 if (!CI || !ConstantIsOk(CI))
6949 return std::nullopt;
6950
6951 if (!SafeReplacementConstant)
6952 SafeReplacementConstant = CI;
6953 }
6954 } else if (isa<VectorType>(C->getType())) {
6955 // Handle scalable splat
6956 Value *SplatC = C->getSplatValue();
6957 auto *CI = dyn_cast_or_null<ConstantInt>(SplatC);
6958 // Bail out if the constant can't be safely incremented/decremented.
6959 if (!CI || !ConstantIsOk(CI))
6960 return std::nullopt;
6961 } else {
6962 // ConstantExpr?
6963 return std::nullopt;
6964 }
6965
6966 // It may not be safe to change a compare predicate in the presence of
6967 // undefined elements, so replace those elements with the first safe constant
6968 // that we found.
6969 // TODO: in case of poison, it is safe; let's replace undefs only.
6970 if (C->containsUndefOrPoisonElement()) {
6971 assert(SafeReplacementConstant && "Replacement constant not set");
6972 C = Constant::replaceUndefsWith(C, SafeReplacementConstant);
6973 }
6974
6976
6977 // Increment or decrement the constant.
6978 Constant *OneOrNegOne = ConstantInt::get(Type, WillIncrement ? 1 : -1, true);
6979 Constant *NewC = ConstantExpr::getAdd(C, OneOrNegOne);
6980
6981 return std::make_pair(NewPred, NewC);
6982}
6983
6984/// If we have an icmp le or icmp ge instruction with a constant operand, turn
6985/// it into the appropriate icmp lt or icmp gt instruction. This transform
6986/// allows them to be folded in visitICmpInst.
6988 ICmpInst::Predicate Pred = I.getPredicate();
6989 if (ICmpInst::isEquality(Pred) || !ICmpInst::isIntPredicate(Pred) ||
6991 return nullptr;
6992
6993 Value *Op0 = I.getOperand(0);
6994 Value *Op1 = I.getOperand(1);
6995 auto *Op1C = dyn_cast<Constant>(Op1);
6996 if (!Op1C)
6997 return nullptr;
6998
6999 auto FlippedStrictness =
7001 if (!FlippedStrictness)
7002 return nullptr;
7003
7004 return new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second);
7005}
7006
7007/// If we have a comparison with a non-canonical predicate, if we can update
7008/// all the users, invert the predicate and adjust all the users.
7010 // Is the predicate already canonical?
7011 CmpInst::Predicate Pred = I.getPredicate();
7013 return nullptr;
7014
7015 // Can all users be adjusted to predicate inversion?
7016 if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr))
7017 return nullptr;
7018
7019 // Ok, we can canonicalize comparison!
7020 // Let's first invert the comparison's predicate.
7021 I.setPredicate(CmpInst::getInversePredicate(Pred));
7022 I.setName(I.getName() + ".not");
7023
7024 // And, adapt users.
7026
7027 return &I;
7028}
7029
7030/// Integer compare with boolean values can always be turned into bitwise ops.
7032 InstCombiner::BuilderTy &Builder) {
7033 Value *A = I.getOperand(0), *B = I.getOperand(1);
7034 assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only");
7035
7036 // A boolean compared to true/false can be simplified to Op0/true/false in
7037 // 14 out of the 20 (10 predicates * 2 constants) possible combinations.
7038 // Cases not handled by InstSimplify are always 'not' of Op0.
7039 if (match(B, m_Zero())) {
7040 switch (I.getPredicate()) {
7041 case CmpInst::ICMP_EQ: // A == 0 -> !A
7042 case CmpInst::ICMP_ULE: // A <=u 0 -> !A
7043 case CmpInst::ICMP_SGE: // A >=s 0 -> !A
7045 default:
7046 llvm_unreachable("ICmp i1 X, C not simplified as expected.");
7047 }
7048 } else if (match(B, m_One())) {
7049 switch (I.getPredicate()) {
7050 case CmpInst::ICMP_NE: // A != 1 -> !A
7051 case CmpInst::ICMP_ULT: // A <u 1 -> !A
7052 case CmpInst::ICMP_SGT: // A >s -1 -> !A
7054 default:
7055 llvm_unreachable("ICmp i1 X, C not simplified as expected.");
7056 }
7057 }
7058
7059 switch (I.getPredicate()) {
7060 default:
7061 llvm_unreachable("Invalid icmp instruction!");
7062 case ICmpInst::ICMP_EQ:
7063 // icmp eq i1 A, B -> ~(A ^ B)
7064 return BinaryOperator::CreateNot(Builder.CreateXor(A, B));
7065
7066 case ICmpInst::ICMP_NE:
7067 // icmp ne i1 A, B -> A ^ B
7068 return BinaryOperator::CreateXor(A, B);
7069
7070 case ICmpInst::ICMP_UGT:
7071 // icmp ugt -> icmp ult
7072 std::swap(A, B);
7073 [[fallthrough]];
7074 case ICmpInst::ICMP_ULT:
7075 // icmp ult i1 A, B -> ~A & B
7076 return BinaryOperator::CreateAnd(Builder.CreateNot(A), B);
7077
7078 case ICmpInst::ICMP_SGT:
7079 // icmp sgt -> icmp slt
7080 std::swap(A, B);
7081 [[fallthrough]];
7082 case ICmpInst::ICMP_SLT:
7083 // icmp slt i1 A, B -> A & ~B
7084 return BinaryOperator::CreateAnd(Builder.CreateNot(B), A);
7085
7086 case ICmpInst::ICMP_UGE:
7087 // icmp uge -> icmp ule
7088 std::swap(A, B);
7089 [[fallthrough]];
7090 case ICmpInst::ICMP_ULE:
7091 // icmp ule i1 A, B -> ~A | B
7092 return BinaryOperator::CreateOr(Builder.CreateNot(A), B);
7093
7094 case ICmpInst::ICMP_SGE:
7095 // icmp sge -> icmp sle
7096 std::swap(A, B);
7097 [[fallthrough]];
7098 case ICmpInst::ICMP_SLE:
7099 // icmp sle i1 A, B -> A | ~B
7100 return BinaryOperator::CreateOr(Builder.CreateNot(B), A);
7101 }
7102}
7103
7104// Transform pattern like:
7105// (1 << Y) u<= X or ~(-1 << Y) u< X or ((1 << Y)+(-1)) u< X
7106// (1 << Y) u> X or ~(-1 << Y) u>= X or ((1 << Y)+(-1)) u>= X
7107// Into:
7108// (X l>> Y) != 0
7109// (X l>> Y) == 0
7111 InstCombiner::BuilderTy &Builder) {
7112 CmpPredicate Pred, NewPred;
7113 Value *X, *Y;
7114 if (match(&Cmp,
7115 m_c_ICmp(Pred, m_OneUse(m_Shl(m_One(), m_Value(Y))), m_Value(X)))) {
7116 switch (Pred) {
7117 case ICmpInst::ICMP_ULE:
7118 NewPred = ICmpInst::ICMP_NE;
7119 break;
7120 case ICmpInst::ICMP_UGT:
7121 NewPred = ICmpInst::ICMP_EQ;
7122 break;
7123 default:
7124 return nullptr;
7125 }
7126 } else if (match(&Cmp, m_c_ICmp(Pred,
7129 m_Add(m_Shl(m_One(), m_Value(Y)),
7130 m_AllOnes()))),
7131 m_Value(X)))) {
7132 // The variant with 'add' is not canonical, (the variant with 'not' is)
7133 // we only get it because it has extra uses, and can't be canonicalized,
7134
7135 switch (Pred) {
7136 case ICmpInst::ICMP_ULT:
7137 NewPred = ICmpInst::ICMP_NE;
7138 break;
7139 case ICmpInst::ICMP_UGE:
7140 NewPred = ICmpInst::ICMP_EQ;
7141 break;
7142 default:
7143 return nullptr;
7144 }
7145 } else
7146 return nullptr;
7147
7148 Value *NewX = Builder.CreateLShr(X, Y, X->getName() + ".highbits");
7149 Constant *Zero = Constant::getNullValue(NewX->getType());
7150 return CmpInst::Create(Instruction::ICmp, NewPred, NewX, Zero);
7151}
7152
7154 InstCombiner::BuilderTy &Builder) {
7155 const CmpInst::Predicate Pred = Cmp.getPredicate();
7156 Value *LHS = Cmp.getOperand(0), *RHS = Cmp.getOperand(1);
7157 Value *V1, *V2;
7158
7159 auto createCmpReverse = [&](CmpInst::Predicate Pred, Value *X, Value *Y) {
7160 Value *V = Builder.CreateCmp(Pred, X, Y, Cmp.getName());
7161 if (auto *I = dyn_cast<Instruction>(V))
7162 I->copyIRFlags(&Cmp);
7163 Module *M = Cmp.getModule();
7165 M, Intrinsic::vector_reverse, V->getType());
7166 return CallInst::Create(F, V);
7167 };
7168
7169 if (match(LHS, m_VecReverse(m_Value(V1)))) {
7170 // cmp Pred, rev(V1), rev(V2) --> rev(cmp Pred, V1, V2)
7171 if (match(RHS, m_VecReverse(m_Value(V2))) &&
7172 (LHS->hasOneUse() || RHS->hasOneUse()))
7173 return createCmpReverse(Pred, V1, V2);
7174
7175 // cmp Pred, rev(V1), RHSSplat --> rev(cmp Pred, V1, RHSSplat)
7176 if (LHS->hasOneUse() && isSplatValue(RHS))
7177 return createCmpReverse(Pred, V1, RHS);
7178 }
7179 // cmp Pred, LHSSplat, rev(V2) --> rev(cmp Pred, LHSSplat, V2)
7180 else if (isSplatValue(LHS) && match(RHS, m_OneUse(m_VecReverse(m_Value(V2)))))
7181 return createCmpReverse(Pred, LHS, V2);
7182
7183 ArrayRef<int> M;
7184 if (!match(LHS, m_Shuffle(m_Value(V1), m_Undef(), m_Mask(M))))
7185 return nullptr;
7186
7187 // If both arguments of the cmp are shuffles that use the same mask and
7188 // shuffle within a single vector, move the shuffle after the cmp:
7189 // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M
7190 Type *V1Ty = V1->getType();
7191 if (match(RHS, m_Shuffle(m_Value(V2), m_Undef(), m_SpecificMask(M))) &&
7192 V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) {
7193 Value *NewCmp = Builder.CreateCmp(Pred, V1, V2);
7194 return new ShuffleVectorInst(NewCmp, M);
7195 }
7196
7197 // Try to canonicalize compare with splatted operand and splat constant.
7198 // TODO: We could generalize this for more than splats. See/use the code in
7199 // InstCombiner::foldVectorBinop().
7200 Constant *C;
7201 if (!LHS->hasOneUse() || !match(RHS, m_Constant(C)))
7202 return nullptr;
7203
7204 // Length-changing splats are ok, so adjust the constants as needed:
7205 // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M
7206 Constant *ScalarC = C->getSplatValue(/* AllowPoison */ true);
7207 int MaskSplatIndex;
7208 if (ScalarC && match(M, m_SplatOrPoisonMask(MaskSplatIndex))) {
7209 // We allow poison in matching, but this transform removes it for safety.
7210 // Demanded elements analysis should be able to recover some/all of that.
7211 C = ConstantVector::getSplat(cast<VectorType>(V1Ty)->getElementCount(),
7212 ScalarC);
7213 SmallVector<int, 8> NewM(M.size(), MaskSplatIndex);
7214 Value *NewCmp = Builder.CreateCmp(Pred, V1, C);
7215 return new ShuffleVectorInst(NewCmp, NewM);
7216 }
7217
7218 return nullptr;
7219}
7220
7221// extract(uadd.with.overflow(A, B), 0) ult A
7222// -> extract(uadd.with.overflow(A, B), 1)
7224 CmpInst::Predicate Pred = I.getPredicate();
7225 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
7226
7227 Value *UAddOv;
7228 Value *A, *B;
7229 auto UAddOvResultPat = m_ExtractValue<0>(
7230 m_Intrinsic<Intrinsic::uadd_with_overflow>(m_Value(A), m_Value(B)));
7231 if (match(Op0, UAddOvResultPat) &&
7232 ((Pred == ICmpInst::ICMP_ULT && (Op1 == A || Op1 == B)) ||
7233 (Pred == ICmpInst::ICMP_EQ && match(Op1, m_ZeroInt()) &&
7234 (match(A, m_One()) || match(B, m_One()))) ||
7235 (Pred == ICmpInst::ICMP_NE && match(Op1, m_AllOnes()) &&
7236 (match(A, m_AllOnes()) || match(B, m_AllOnes())))))
7237 // extract(uadd.with.overflow(A, B), 0) < A
7238 // extract(uadd.with.overflow(A, 1), 0) == 0
7239 // extract(uadd.with.overflow(A, -1), 0) != -1
7240 UAddOv = cast<ExtractValueInst>(Op0)->getAggregateOperand();
7241 else if (match(Op1, UAddOvResultPat) &&
7242 Pred == ICmpInst::ICMP_UGT && (Op0 == A || Op0 == B))
7243 // A > extract(uadd.with.overflow(A, B), 0)
7244 UAddOv = cast<ExtractValueInst>(Op1)->getAggregateOperand();
7245 else
7246 return nullptr;
7247
7248 return ExtractValueInst::Create(UAddOv, 1);
7249}
7250
7252 if (!I.getOperand(0)->getType()->isPointerTy() ||
7254 I.getParent()->getParent(),
7255 I.getOperand(0)->getType()->getPointerAddressSpace())) {
7256 return nullptr;
7257 }
7258 Instruction *Op;
7259 if (match(I.getOperand(0), m_Instruction(Op)) &&
7260 match(I.getOperand(1), m_Zero()) &&
7261 Op->isLaunderOrStripInvariantGroup()) {
7262 return ICmpInst::Create(Instruction::ICmp, I.getPredicate(),
7263 Op->getOperand(0), I.getOperand(1));
7264 }
7265 return nullptr;
7266}
7267
7268/// This function folds patterns produced by lowering of reduce idioms, such as
7269/// llvm.vector.reduce.and which are lowered into instruction chains. This code
7270/// attempts to generate fewer number of scalar comparisons instead of vector
7271/// comparisons when possible.
7273 InstCombiner::BuilderTy &Builder,
7274 const DataLayout &DL) {
7275 if (I.getType()->isVectorTy())
7276 return nullptr;
7277 CmpPredicate OuterPred, InnerPred;
7278 Value *LHS, *RHS;
7279
7280 // Match lowering of @llvm.vector.reduce.and. Turn
7281 /// %vec_ne = icmp ne <8 x i8> %lhs, %rhs
7282 /// %scalar_ne = bitcast <8 x i1> %vec_ne to i8
7283 /// %res = icmp <pred> i8 %scalar_ne, 0
7284 ///
7285 /// into
7286 ///
7287 /// %lhs.scalar = bitcast <8 x i8> %lhs to i64
7288 /// %rhs.scalar = bitcast <8 x i8> %rhs to i64
7289 /// %res = icmp <pred> i64 %lhs.scalar, %rhs.scalar
7290 ///
7291 /// for <pred> in {ne, eq}.
7292 if (!match(&I, m_ICmp(OuterPred,
7294 m_ICmp(InnerPred, m_Value(LHS), m_Value(RHS))))),
7295 m_Zero())))
7296 return nullptr;
7297 auto *LHSTy = dyn_cast<FixedVectorType>(LHS->getType());
7298 if (!LHSTy || !LHSTy->getElementType()->isIntegerTy())
7299 return nullptr;
7300 unsigned NumBits =
7301 LHSTy->getNumElements() * LHSTy->getElementType()->getIntegerBitWidth();
7302 // TODO: Relax this to "not wider than max legal integer type"?
7303 if (!DL.isLegalInteger(NumBits))
7304 return nullptr;
7305
7306 if (ICmpInst::isEquality(OuterPred) && InnerPred == ICmpInst::ICMP_NE) {
7307 auto *ScalarTy = Builder.getIntNTy(NumBits);
7308 LHS = Builder.CreateBitCast(LHS, ScalarTy, LHS->getName() + ".scalar");
7309 RHS = Builder.CreateBitCast(RHS, ScalarTy, RHS->getName() + ".scalar");
7310 return ICmpInst::Create(Instruction::ICmp, OuterPred, LHS, RHS,
7311 I.getName());
7312 }
7313
7314 return nullptr;
7315}
7316
7317// This helper will be called with icmp operands in both orders.
7319 Value *Op0, Value *Op1,
7320 ICmpInst &CxtI) {
7321 // Try to optimize 'icmp GEP, P' or 'icmp P, GEP'.
7322 if (auto *GEP = dyn_cast<GEPOperator>(Op0))
7323 if (Instruction *NI = foldGEPICmp(GEP, Op1, Pred, CxtI))
7324 return NI;
7325
7326 if (auto *SI = dyn_cast<SelectInst>(Op0))
7327 if (Instruction *NI = foldSelectICmp(Pred, SI, Op1, CxtI))
7328 return NI;
7329
7330 if (auto *MinMax = dyn_cast<MinMaxIntrinsic>(Op0))
7331 if (Instruction *Res = foldICmpWithMinMax(CxtI, MinMax, Op1, Pred))
7332 return Res;
7333
7334 {
7335 Value *X;
7336 const APInt *C;
7337 // icmp X+Cst, X
7338 if (match(Op0, m_Add(m_Value(X), m_APInt(C))) && Op1 == X)
7339 return foldICmpAddOpConst(X, *C, Pred);
7340 }
7341
7342 // abs(X) >= X --> true
7343 // abs(X) u<= X --> true
7344 // abs(X) < X --> false
7345 // abs(X) u> X --> false
7346 // abs(X) u>= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
7347 // abs(X) <= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
7348 // abs(X) == X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
7349 // abs(X) u< X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
7350 // abs(X) > X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
7351 // abs(X) != X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
7352 {
7353 Value *X;
7354 Constant *C;
7355 if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(X), m_Constant(C))) &&
7356 match(Op1, m_Specific(X))) {
7357 Value *NullValue = Constant::getNullValue(X->getType());
7358 Value *AllOnesValue = Constant::getAllOnesValue(X->getType());
7359 const APInt SMin =
7360 APInt::getSignedMinValue(X->getType()->getScalarSizeInBits());
7361 bool IsIntMinPosion = C->isAllOnesValue();
7362 switch (Pred) {
7363 case CmpInst::ICMP_ULE:
7364 case CmpInst::ICMP_SGE:
7365 return replaceInstUsesWith(CxtI, ConstantInt::getTrue(CxtI.getType()));
7366 case CmpInst::ICMP_UGT:
7367 case CmpInst::ICMP_SLT:
7369 case CmpInst::ICMP_UGE:
7370 case CmpInst::ICMP_SLE:
7371 case CmpInst::ICMP_EQ: {
7372 return replaceInstUsesWith(
7373 CxtI, IsIntMinPosion
7374 ? Builder.CreateICmpSGT(X, AllOnesValue)
7376 X, ConstantInt::get(X->getType(), SMin + 1)));
7377 }
7378 case CmpInst::ICMP_ULT:
7379 case CmpInst::ICMP_SGT:
7380 case CmpInst::ICMP_NE: {
7381 return replaceInstUsesWith(
7382 CxtI, IsIntMinPosion
7383 ? Builder.CreateICmpSLT(X, NullValue)
7385 X, ConstantInt::get(X->getType(), SMin)));
7386 }
7387 default:
7388 llvm_unreachable("Invalid predicate!");
7389 }
7390 }
7391 }
7392
7393 const SimplifyQuery Q = SQ.getWithInstruction(&CxtI);
7394 if (Value *V = foldICmpWithLowBitMaskedVal(Pred, Op0, Op1, Q, *this))
7395 return replaceInstUsesWith(CxtI, V);
7396
7397 // Folding (X / Y) pred X => X swap(pred) 0 for constant Y other than 0 or 1
7398 auto CheckUGT1 = [](const APInt &Divisor) { return Divisor.ugt(1); };
7399 {
7400 if (match(Op0, m_UDiv(m_Specific(Op1), m_CheckedInt(CheckUGT1)))) {
7401 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
7403 }
7404
7405 if (!ICmpInst::isUnsigned(Pred) &&
7406 match(Op0, m_SDiv(m_Specific(Op1), m_CheckedInt(CheckUGT1)))) {
7407 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
7409 }
7410 }
7411
7412 // Another case of this fold is (X >> Y) pred X => X swap(pred) 0 if Y != 0
7413 auto CheckNE0 = [](const APInt &Shift) { return !Shift.isZero(); };
7414 {
7415 if (match(Op0, m_LShr(m_Specific(Op1), m_CheckedInt(CheckNE0)))) {
7416 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
7418 }
7419
7420 if ((Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SGE) &&
7421 match(Op0, m_AShr(m_Specific(Op1), m_CheckedInt(CheckNE0)))) {
7422 return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1,
7424 }
7425 }
7426
7427 return nullptr;
7428}
7429
7431 bool Changed = false;
7433 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
7434 unsigned Op0Cplxity = getComplexity(Op0);
7435 unsigned Op1Cplxity = getComplexity(Op1);
7436
7437 /// Orders the operands of the compare so that they are listed from most
7438 /// complex to least complex. This puts constants before unary operators,
7439 /// before binary operators.
7440 if (Op0Cplxity < Op1Cplxity) {
7441 I.swapOperands();
7442 std::swap(Op0, Op1);
7443 Changed = true;
7444 }
7445
7446 if (Value *V = simplifyICmpInst(I.getCmpPredicate(), Op0, Op1, Q))
7447 return replaceInstUsesWith(I, V);
7448
7449 // Comparing -val or val with non-zero is the same as just comparing val
7450 // ie, abs(val) != 0 -> val != 0
7451 if (I.getPredicate() == ICmpInst::ICMP_NE && match(Op1, m_Zero())) {
7452 Value *Cond, *SelectTrue, *SelectFalse;
7453 if (match(Op0, m_Select(m_Value(Cond), m_Value(SelectTrue),
7454 m_Value(SelectFalse)))) {
7455 if (Value *V = dyn_castNegVal(SelectTrue)) {
7456 if (V == SelectFalse)
7457 return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1);
7458 }
7459 else if (Value *V = dyn_castNegVal(SelectFalse)) {
7460 if (V == SelectTrue)
7461 return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1);
7462 }
7463 }
7464 }
7465
7466 if (Op0->getType()->isIntOrIntVectorTy(1))
7468 return Res;
7469
7471 return Res;
7472
7474 return Res;
7475
7477 return Res;
7478
7480 return Res;
7481
7483 return Res;
7484
7486 return Res;
7487
7489 return Res;
7490
7491 // Test if the ICmpInst instruction is used exclusively by a select as
7492 // part of a minimum or maximum operation. If so, refrain from doing
7493 // any other folding. This helps out other analyses which understand
7494 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
7495 // and CodeGen. And in this case, at least one of the comparison
7496 // operands has at least one user besides the compare (the select),
7497 // which would often largely negate the benefit of folding anyway.
7498 //
7499 // Do the same for the other patterns recognized by matchSelectPattern.
7500 if (I.hasOneUse())
7501 if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) {
7502 Value *A, *B;
7504 if (SPR.Flavor != SPF_UNKNOWN)
7505 return nullptr;
7506 }
7507
7508 // Do this after checking for min/max to prevent infinite looping.
7509 if (Instruction *Res = foldICmpWithZero(I))
7510 return Res;
7511
7512 // FIXME: We only do this after checking for min/max to prevent infinite
7513 // looping caused by a reverse canonicalization of these patterns for min/max.
7514 // FIXME: The organization of folds is a mess. These would naturally go into
7515 // canonicalizeCmpWithConstant(), but we can't move all of the above folds
7516 // down here after the min/max restriction.
7517 ICmpInst::Predicate Pred = I.getPredicate();
7518 const APInt *C;
7519 if (match(Op1, m_APInt(C))) {
7520 // For i32: x >u 2147483647 -> x <s 0 -> true if sign bit set
7521 if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) {
7522 Constant *Zero = Constant::getNullValue(Op0->getType());
7523 return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero);
7524 }
7525
7526 // For i32: x <u 2147483648 -> x >s -1 -> true if sign bit clear
7527 if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) {
7529 return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes);
7530 }
7531 }
7532
7533 // The folds in here may rely on wrapping flags and special constants, so
7534 // they can break up min/max idioms in some cases but not seemingly similar
7535 // patterns.
7536 // FIXME: It may be possible to enhance select folding to make this
7537 // unnecessary. It may also be moot if we canonicalize to min/max
7538 // intrinsics.
7539 if (Instruction *Res = foldICmpBinOp(I, Q))
7540 return Res;
7541
7543 return Res;
7544
7545 // Try to match comparison as a sign bit test. Intentionally do this after
7546 // foldICmpInstWithConstant() to potentially let other folds to happen first.
7547 if (Instruction *New = foldSignBitTest(I))
7548 return New;
7549
7551 return Res;
7552
7553 if (Instruction *Res = foldICmpCommutative(I.getCmpPredicate(), Op0, Op1, I))
7554 return Res;
7555 if (Instruction *Res =
7556 foldICmpCommutative(I.getSwappedCmpPredicate(), Op1, Op0, I))
7557 return Res;
7558
7559 if (I.isCommutative()) {
7560 if (auto Pair = matchSymmetricPair(I.getOperand(0), I.getOperand(1))) {
7561 replaceOperand(I, 0, Pair->first);
7562 replaceOperand(I, 1, Pair->second);
7563 return &I;
7564 }
7565 }
7566
7567 // In case of a comparison with two select instructions having the same
7568 // condition, check whether one of the resulting branches can be simplified.
7569 // If so, just compare the other branch and select the appropriate result.
7570 // For example:
7571 // %tmp1 = select i1 %cmp, i32 %y, i32 %x
7572 // %tmp2 = select i1 %cmp, i32 %z, i32 %x
7573 // %cmp2 = icmp slt i32 %tmp2, %tmp1
7574 // The icmp will result false for the false value of selects and the result
7575 // will depend upon the comparison of true values of selects if %cmp is
7576 // true. Thus, transform this into:
7577 // %cmp = icmp slt i32 %y, %z
7578 // %sel = select i1 %cond, i1 %cmp, i1 false
7579 // This handles similar cases to transform.
7580 {
7581 Value *Cond, *A, *B, *C, *D;
7582 if (match(Op0, m_Select(m_Value(Cond), m_Value(A), m_Value(B))) &&
7584 (Op0->hasOneUse() || Op1->hasOneUse())) {
7585 // Check whether comparison of TrueValues can be simplified
7586 if (Value *Res = simplifyICmpInst(Pred, A, C, SQ)) {
7587 Value *NewICMP = Builder.CreateICmp(Pred, B, D);
7588 return SelectInst::Create(Cond, Res, NewICMP);
7589 }
7590 // Check whether comparison of FalseValues can be simplified
7591 if (Value *Res = simplifyICmpInst(Pred, B, D, SQ)) {
7592 Value *NewICMP = Builder.CreateICmp(Pred, A, C);
7593 return SelectInst::Create(Cond, NewICMP, Res);
7594 }
7595 }
7596 }
7597
7598 // Try to optimize equality comparisons against alloca-based pointers.
7599 if (Op0->getType()->isPointerTy() && I.isEquality()) {
7600 assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?");
7601 if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0)))
7602 if (foldAllocaCmp(Alloca))
7603 return nullptr;
7604 if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1)))
7605 if (foldAllocaCmp(Alloca))
7606 return nullptr;
7607 }
7608
7609 if (Instruction *Res = foldICmpBitCast(I))
7610 return Res;
7611
7612 // TODO: Hoist this above the min/max bailout.
7614 return R;
7615
7616 {
7617 Value *X, *Y;
7618 // Transform (X & ~Y) == 0 --> (X & Y) != 0
7619 // and (X & ~Y) != 0 --> (X & Y) == 0
7620 // if A is a power of 2.
7621 if (match(Op0, m_And(m_Value(X), m_Not(m_Value(Y)))) &&
7622 match(Op1, m_Zero()) && isKnownToBeAPowerOfTwo(X, false, 0, &I) &&
7623 I.isEquality())
7624 return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(X, Y),
7625 Op1);
7626
7627 // Op0 pred Op1 -> ~Op1 pred ~Op0, if this allows us to drop an instruction.
7628 if (Op0->getType()->isIntOrIntVectorTy()) {
7629 bool ConsumesOp0, ConsumesOp1;
7630 if (isFreeToInvert(Op0, Op0->hasOneUse(), ConsumesOp0) &&
7631 isFreeToInvert(Op1, Op1->hasOneUse(), ConsumesOp1) &&
7632 (ConsumesOp0 || ConsumesOp1)) {
7633 Value *InvOp0 = getFreelyInverted(Op0, Op0->hasOneUse(), &Builder);
7634 Value *InvOp1 = getFreelyInverted(Op1, Op1->hasOneUse(), &Builder);
7635 assert(InvOp0 && InvOp1 &&
7636 "Mismatch between isFreeToInvert and getFreelyInverted");
7637 return new ICmpInst(I.getSwappedPredicate(), InvOp0, InvOp1);
7638 }
7639 }
7640
7641 Instruction *AddI = nullptr;
7643 m_Instruction(AddI))) &&
7644 isa<IntegerType>(X->getType())) {
7645 Value *Result;
7646 Constant *Overflow;
7647 // m_UAddWithOverflow can match patterns that do not include an explicit
7648 // "add" instruction, so check the opcode of the matched op.
7649 if (AddI->getOpcode() == Instruction::Add &&
7650 OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, X, Y, *AddI,
7651 Result, Overflow)) {
7652 replaceInstUsesWith(*AddI, Result);
7653 eraseInstFromFunction(*AddI);
7654 return replaceInstUsesWith(I, Overflow);
7655 }
7656 }
7657
7658 // (zext X) * (zext Y) --> llvm.umul.with.overflow.
7659 if (match(Op0, m_NUWMul(m_ZExt(m_Value(X)), m_ZExt(m_Value(Y)))) &&
7660 match(Op1, m_APInt(C))) {
7661 if (Instruction *R = processUMulZExtIdiom(I, Op0, C, *this))
7662 return R;
7663 }
7664
7665 // Signbit test folds
7666 // Fold (X u>> BitWidth - 1 Pred ZExt(i1)) --> X s< 0 Pred i1
7667 // Fold (X s>> BitWidth - 1 Pred SExt(i1)) --> X s< 0 Pred i1
7668 Instruction *ExtI;
7669 if ((I.isUnsigned() || I.isEquality()) &&
7670 match(Op1,
7672 Y->getType()->getScalarSizeInBits() == 1 &&
7673 (Op0->hasOneUse() || Op1->hasOneUse())) {
7674 unsigned OpWidth = Op0->getType()->getScalarSizeInBits();
7675 Instruction *ShiftI;
7676 if (match(Op0, m_CombineAnd(m_Instruction(ShiftI),
7678 OpWidth - 1))))) {
7679 unsigned ExtOpc = ExtI->getOpcode();
7680 unsigned ShiftOpc = ShiftI->getOpcode();
7681 if ((ExtOpc == Instruction::ZExt && ShiftOpc == Instruction::LShr) ||
7682 (ExtOpc == Instruction::SExt && ShiftOpc == Instruction::AShr)) {
7683 Value *SLTZero =
7685 Value *Cmp = Builder.CreateICmp(Pred, SLTZero, Y, I.getName());
7686 return replaceInstUsesWith(I, Cmp);
7687 }
7688 }
7689 }
7690 }
7691
7692 if (Instruction *Res = foldICmpEquality(I))
7693 return Res;
7694
7696 return Res;
7697
7698 if (Instruction *Res = foldICmpOfUAddOv(I))
7699 return Res;
7700
7701 // The 'cmpxchg' instruction returns an aggregate containing the old value and
7702 // an i1 which indicates whether or not we successfully did the swap.
7703 //
7704 // Replace comparisons between the old value and the expected value with the
7705 // indicator that 'cmpxchg' returns.
7706 //
7707 // N.B. This transform is only valid when the 'cmpxchg' is not permitted to
7708 // spuriously fail. In those cases, the old value may equal the expected
7709 // value but it is possible for the swap to not occur.
7710 if (I.getPredicate() == ICmpInst::ICMP_EQ)
7711 if (auto *EVI = dyn_cast<ExtractValueInst>(Op0))
7712 if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(EVI->getAggregateOperand()))
7713 if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 &&
7714 !ACXI->isWeak())
7715 return ExtractValueInst::Create(ACXI, 1);
7716
7718 return Res;
7719
7720 if (I.getType()->isVectorTy())
7721 if (Instruction *Res = foldVectorCmp(I, Builder))
7722 return Res;
7723
7725 return Res;
7726
7728 return Res;
7729
7730 {
7731 Value *A;
7732 const APInt *C1, *C2;
7733 ICmpInst::Predicate Pred = I.getPredicate();
7734 if (ICmpInst::isEquality(Pred)) {
7735 // sext(a) & c1 == c2 --> a & c3 == trunc(c2)
7736 // sext(a) & c1 != c2 --> a & c3 != trunc(c2)
7737 if (match(Op0, m_And(m_SExt(m_Value(A)), m_APInt(C1))) &&
7738 match(Op1, m_APInt(C2))) {
7739 Type *InputTy = A->getType();
7740 unsigned InputBitWidth = InputTy->getScalarSizeInBits();
7741 // c2 must be non-negative at the bitwidth of a.
7742 if (C2->getActiveBits() < InputBitWidth) {
7743 APInt TruncC1 = C1->trunc(InputBitWidth);
7744 // Check if there are 1s in C1 high bits of size InputBitWidth.
7745 if (C1->uge(APInt::getOneBitSet(C1->getBitWidth(), InputBitWidth)))
7746 TruncC1.setBit(InputBitWidth - 1);
7747 Value *AndInst = Builder.CreateAnd(A, TruncC1);
7748 return new ICmpInst(
7749 Pred, AndInst,
7750 ConstantInt::get(InputTy, C2->trunc(InputBitWidth)));
7751 }
7752 }
7753 }
7754 }
7755
7756 return Changed ? &I : nullptr;
7757}
7758
7759/// Fold fcmp ([us]itofp x, cst) if possible.
7761 Instruction *LHSI,
7762 Constant *RHSC) {
7763 const APFloat *RHS;
7764 if (!match(RHSC, m_APFloat(RHS)))
7765 return nullptr;
7766
7767 // Get the width of the mantissa. We don't want to hack on conversions that
7768 // might lose information from the integer, e.g. "i64 -> float"
7769 int MantissaWidth = LHSI->getType()->getFPMantissaWidth();
7770 if (MantissaWidth == -1) return nullptr; // Unknown.
7771
7772 Type *IntTy = LHSI->getOperand(0)->getType();
7773 unsigned IntWidth = IntTy->getScalarSizeInBits();
7774 bool LHSUnsigned = isa<UIToFPInst>(LHSI);
7775
7776 if (I.isEquality()) {
7777 FCmpInst::Predicate P = I.getPredicate();
7778 bool IsExact = false;
7779 APSInt RHSCvt(IntWidth, LHSUnsigned);
7780 RHS->convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact);
7781
7782 // If the floating point constant isn't an integer value, we know if we will
7783 // ever compare equal / not equal to it.
7784 if (!IsExact) {
7785 // TODO: Can never be -0.0 and other non-representable values
7786 APFloat RHSRoundInt(*RHS);
7788 if (*RHS != RHSRoundInt) {
7790 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7791
7793 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7794 }
7795 }
7796
7797 // TODO: If the constant is exactly representable, is it always OK to do
7798 // equality compares as integer?
7799 }
7800
7801 // Check to see that the input is converted from an integer type that is small
7802 // enough that preserves all bits. TODO: check here for "known" sign bits.
7803 // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e.
7804
7805 // Following test does NOT adjust IntWidth downwards for signed inputs,
7806 // because the most negative value still requires all the mantissa bits
7807 // to distinguish it from one less than that value.
7808 if ((int)IntWidth > MantissaWidth) {
7809 // Conversion would lose accuracy. Check if loss can impact comparison.
7810 int Exp = ilogb(*RHS);
7811 if (Exp == APFloat::IEK_Inf) {
7812 int MaxExponent = ilogb(APFloat::getLargest(RHS->getSemantics()));
7813 if (MaxExponent < (int)IntWidth - !LHSUnsigned)
7814 // Conversion could create infinity.
7815 return nullptr;
7816 } else {
7817 // Note that if RHS is zero or NaN, then Exp is negative
7818 // and first condition is trivially false.
7819 if (MantissaWidth <= Exp && Exp <= (int)IntWidth - !LHSUnsigned)
7820 // Conversion could affect comparison.
7821 return nullptr;
7822 }
7823 }
7824
7825 // Otherwise, we can potentially simplify the comparison. We know that it
7826 // will always come through as an integer value and we know the constant is
7827 // not a NAN (it would have been previously simplified).
7828 assert(!RHS->isNaN() && "NaN comparison not already folded!");
7829
7831 switch (I.getPredicate()) {
7832 default: llvm_unreachable("Unexpected predicate!");
7833 case FCmpInst::FCMP_UEQ:
7834 case FCmpInst::FCMP_OEQ:
7835 Pred = ICmpInst::ICMP_EQ;
7836 break;
7837 case FCmpInst::FCMP_UGT:
7838 case FCmpInst::FCMP_OGT:
7839 Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT;
7840 break;
7841 case FCmpInst::FCMP_UGE:
7842 case FCmpInst::FCMP_OGE:
7843 Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE;
7844 break;
7845 case FCmpInst::FCMP_ULT:
7846 case FCmpInst::FCMP_OLT:
7847 Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT;
7848 break;
7849 case FCmpInst::FCMP_ULE:
7850 case FCmpInst::FCMP_OLE:
7851 Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE;
7852 break;
7853 case FCmpInst::FCMP_UNE:
7854 case FCmpInst::FCMP_ONE:
7855 Pred = ICmpInst::ICMP_NE;
7856 break;
7857 case FCmpInst::FCMP_ORD:
7858 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7859 case FCmpInst::FCMP_UNO:
7860 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7861 }
7862
7863 // Now we know that the APFloat is a normal number, zero or inf.
7864
7865 // See if the FP constant is too large for the integer. For example,
7866 // comparing an i8 to 300.0.
7867 if (!LHSUnsigned) {
7868 // If the RHS value is > SignedMax, fold the comparison. This handles +INF
7869 // and large values.
7870 APFloat SMax(RHS->getSemantics());
7871 SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true,
7873 if (SMax < *RHS) { // smax < 13123.0
7874 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT ||
7875 Pred == ICmpInst::ICMP_SLE)
7876 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7877 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7878 }
7879 } else {
7880 // If the RHS value is > UnsignedMax, fold the comparison. This handles
7881 // +INF and large values.
7882 APFloat UMax(RHS->getSemantics());
7883 UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false,
7885 if (UMax < *RHS) { // umax < 13123.0
7886 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT ||
7887 Pred == ICmpInst::ICMP_ULE)
7888 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7889 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7890 }
7891 }
7892
7893 if (!LHSUnsigned) {
7894 // See if the RHS value is < SignedMin.
7895 APFloat SMin(RHS->getSemantics());
7896 SMin.convertFromAPInt(APInt::getSignedMinValue(IntWidth), true,
7898 if (SMin > *RHS) { // smin > 12312.0
7899 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT ||
7900 Pred == ICmpInst::ICMP_SGE)
7901 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7902 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7903 }
7904 } else {
7905 // See if the RHS value is < UnsignedMin.
7906 APFloat UMin(RHS->getSemantics());
7907 UMin.convertFromAPInt(APInt::getMinValue(IntWidth), false,
7909 if (UMin > *RHS) { // umin > 12312.0
7910 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT ||
7911 Pred == ICmpInst::ICMP_UGE)
7912 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7913 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7914 }
7915 }
7916
7917 // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or
7918 // [0, UMAX], but it may still be fractional. Check whether this is the case
7919 // using the IsExact flag.
7920 // Don't do this for zero, because -0.0 is not fractional.
7921 APSInt RHSInt(IntWidth, LHSUnsigned);
7922 bool IsExact;
7923 RHS->convertToInteger(RHSInt, APFloat::rmTowardZero, &IsExact);
7924 if (!RHS->isZero()) {
7925 if (!IsExact) {
7926 // If we had a comparison against a fractional value, we have to adjust
7927 // the compare predicate and sometimes the value. RHSC is rounded towards
7928 // zero at this point.
7929 switch (Pred) {
7930 default: llvm_unreachable("Unexpected integer comparison!");
7931 case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true
7932 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7933 case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false
7934 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7935 case ICmpInst::ICMP_ULE:
7936 // (float)int <= 4.4 --> int <= 4
7937 // (float)int <= -4.4 --> false
7938 if (RHS->isNegative())
7939 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7940 break;
7941 case ICmpInst::ICMP_SLE:
7942 // (float)int <= 4.4 --> int <= 4
7943 // (float)int <= -4.4 --> int < -4
7944 if (RHS->isNegative())
7945 Pred = ICmpInst::ICMP_SLT;
7946 break;
7947 case ICmpInst::ICMP_ULT:
7948 // (float)int < -4.4 --> false
7949 // (float)int < 4.4 --> int <= 4
7950 if (RHS->isNegative())
7951 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
7952 Pred = ICmpInst::ICMP_ULE;
7953 break;
7954 case ICmpInst::ICMP_SLT:
7955 // (float)int < -4.4 --> int < -4
7956 // (float)int < 4.4 --> int <= 4
7957 if (!RHS->isNegative())
7958 Pred = ICmpInst::ICMP_SLE;
7959 break;
7960 case ICmpInst::ICMP_UGT:
7961 // (float)int > 4.4 --> int > 4
7962 // (float)int > -4.4 --> true
7963 if (RHS->isNegative())
7964 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7965 break;
7966 case ICmpInst::ICMP_SGT:
7967 // (float)int > 4.4 --> int > 4
7968 // (float)int > -4.4 --> int >= -4
7969 if (RHS->isNegative())
7970 Pred = ICmpInst::ICMP_SGE;
7971 break;
7972 case ICmpInst::ICMP_UGE:
7973 // (float)int >= -4.4 --> true
7974 // (float)int >= 4.4 --> int > 4
7975 if (RHS->isNegative())
7976 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
7977 Pred = ICmpInst::ICMP_UGT;
7978 break;
7979 case ICmpInst::ICMP_SGE:
7980 // (float)int >= -4.4 --> int >= -4
7981 // (float)int >= 4.4 --> int > 4
7982 if (!RHS->isNegative())
7983 Pred = ICmpInst::ICMP_SGT;
7984 break;
7985 }
7986 }
7987 }
7988
7989 // Lower this FP comparison into an appropriate integer version of the
7990 // comparison.
7991 return new ICmpInst(Pred, LHSI->getOperand(0),
7992 ConstantInt::get(LHSI->getOperand(0)->getType(), RHSInt));
7993}
7994
7995/// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary.
7997 Constant *RHSC) {
7998 // When C is not 0.0 and infinities are not allowed:
7999 // (C / X) < 0.0 is a sign-bit test of X
8000 // (C / X) < 0.0 --> X < 0.0 (if C is positive)
8001 // (C / X) < 0.0 --> X > 0.0 (if C is negative, swap the predicate)
8002 //
8003 // Proof:
8004 // Multiply (C / X) < 0.0 by X * X / C.
8005 // - X is non zero, if it is the flag 'ninf' is violated.
8006 // - C defines the sign of X * X * C. Thus it also defines whether to swap
8007 // the predicate. C is also non zero by definition.
8008 //
8009 // Thus X * X / C is non zero and the transformation is valid. [qed]
8010
8011 FCmpInst::Predicate Pred = I.getPredicate();
8012
8013 // Check that predicates are valid.
8014 if ((Pred != FCmpInst::FCMP_OGT) && (Pred != FCmpInst::FCMP_OLT) &&
8015 (Pred != FCmpInst::FCMP_OGE) && (Pred != FCmpInst::FCMP_OLE))
8016 return nullptr;
8017
8018 // Check that RHS operand is zero.
8019 if (!match(RHSC, m_AnyZeroFP()))
8020 return nullptr;
8021
8022 // Check fastmath flags ('ninf').
8023 if (!LHSI->hasNoInfs() || !I.hasNoInfs())
8024 return nullptr;
8025
8026 // Check the properties of the dividend. It must not be zero to avoid a
8027 // division by zero (see Proof).
8028 const APFloat *C;
8029 if (!match(LHSI->getOperand(0), m_APFloat(C)))
8030 return nullptr;
8031
8032 if (C->isZero())
8033 return nullptr;
8034
8035 // Get swapped predicate if necessary.
8036 if (C->isNegative())
8037 Pred = I.getSwappedPredicate();
8038
8039 return new FCmpInst(Pred, LHSI->getOperand(1), RHSC, "", &I);
8040}
8041
8042/// Optimize fabs(X) compared with zero.
8044 Value *X;
8045 if (!match(I.getOperand(0), m_FAbs(m_Value(X))))
8046 return nullptr;
8047
8048 const APFloat *C;
8049 if (!match(I.getOperand(1), m_APFloat(C)))
8050 return nullptr;
8051
8052 if (!C->isPosZero()) {
8053 if (!C->isSmallestNormalized())
8054 return nullptr;
8055
8056 const Function *F = I.getFunction();
8057 DenormalMode Mode = F->getDenormalMode(C->getSemantics());
8058 if (Mode.Input == DenormalMode::PreserveSign ||
8059 Mode.Input == DenormalMode::PositiveZero) {
8060
8061 auto replaceFCmp = [](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
8062 Constant *Zero = ConstantFP::getZero(X->getType());
8063 return new FCmpInst(P, X, Zero, "", I);
8064 };
8065
8066 switch (I.getPredicate()) {
8067 case FCmpInst::FCMP_OLT:
8068 // fcmp olt fabs(x), smallest_normalized_number -> fcmp oeq x, 0.0
8069 return replaceFCmp(&I, FCmpInst::FCMP_OEQ, X);
8070 case FCmpInst::FCMP_UGE:
8071 // fcmp uge fabs(x), smallest_normalized_number -> fcmp une x, 0.0
8072 return replaceFCmp(&I, FCmpInst::FCMP_UNE, X);
8073 case FCmpInst::FCMP_OGE:
8074 // fcmp oge fabs(x), smallest_normalized_number -> fcmp one x, 0.0
8075 return replaceFCmp(&I, FCmpInst::FCMP_ONE, X);
8076 case FCmpInst::FCMP_ULT:
8077 // fcmp ult fabs(x), smallest_normalized_number -> fcmp ueq x, 0.0
8078 return replaceFCmp(&I, FCmpInst::FCMP_UEQ, X);
8079 default:
8080 break;
8081 }
8082 }
8083
8084 return nullptr;
8085 }
8086
8087 auto replacePredAndOp0 = [&IC](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
8088 I->setPredicate(P);
8089 return IC.replaceOperand(*I, 0, X);
8090 };
8091
8092 switch (I.getPredicate()) {
8093 case FCmpInst::FCMP_UGE:
8094 case FCmpInst::FCMP_OLT:
8095 // fabs(X) >= 0.0 --> true
8096 // fabs(X) < 0.0 --> false
8097 llvm_unreachable("fcmp should have simplified");
8098
8099 case FCmpInst::FCMP_OGT:
8100 // fabs(X) > 0.0 --> X != 0.0
8101 return replacePredAndOp0(&I, FCmpInst::FCMP_ONE, X);
8102
8103 case FCmpInst::FCMP_UGT:
8104 // fabs(X) u> 0.0 --> X u!= 0.0
8105 return replacePredAndOp0(&I, FCmpInst::FCMP_UNE, X);
8106
8107 case FCmpInst::FCMP_OLE:
8108 // fabs(X) <= 0.0 --> X == 0.0
8109 return replacePredAndOp0(&I, FCmpInst::FCMP_OEQ, X);
8110
8111 case FCmpInst::FCMP_ULE:
8112 // fabs(X) u<= 0.0 --> X u== 0.0
8113 return replacePredAndOp0(&I, FCmpInst::FCMP_UEQ, X);
8114
8115 case FCmpInst::FCMP_OGE:
8116 // fabs(X) >= 0.0 --> !isnan(X)
8117 assert(!I.hasNoNaNs() && "fcmp should have simplified");
8118 return replacePredAndOp0(&I, FCmpInst::FCMP_ORD, X);
8119
8120 case FCmpInst::FCMP_ULT:
8121 // fabs(X) u< 0.0 --> isnan(X)
8122 assert(!I.hasNoNaNs() && "fcmp should have simplified");
8123 return replacePredAndOp0(&I, FCmpInst::FCMP_UNO, X);
8124
8125 case FCmpInst::FCMP_OEQ:
8126 case FCmpInst::FCMP_UEQ:
8127 case FCmpInst::FCMP_ONE:
8128 case FCmpInst::FCMP_UNE:
8129 case FCmpInst::FCMP_ORD:
8130 case FCmpInst::FCMP_UNO:
8131 // Look through the fabs() because it doesn't change anything but the sign.
8132 // fabs(X) == 0.0 --> X == 0.0,
8133 // fabs(X) != 0.0 --> X != 0.0
8134 // isnan(fabs(X)) --> isnan(X)
8135 // !isnan(fabs(X) --> !isnan(X)
8136 return replacePredAndOp0(&I, I.getPredicate(), X);
8137
8138 default:
8139 return nullptr;
8140 }
8141}
8142
8143/// Optimize sqrt(X) compared with zero.
8145 Value *X;
8146 if (!match(I.getOperand(0), m_Sqrt(m_Value(X))))
8147 return nullptr;
8148
8149 if (!match(I.getOperand(1), m_PosZeroFP()))
8150 return nullptr;
8151
8152 auto ReplacePredAndOp0 = [&](FCmpInst::Predicate P) {
8153 I.setPredicate(P);
8154 return IC.replaceOperand(I, 0, X);
8155 };
8156
8157 // Clear ninf flag if sqrt doesn't have it.
8158 if (!cast<Instruction>(I.getOperand(0))->hasNoInfs())
8159 I.setHasNoInfs(false);
8160
8161 switch (I.getPredicate()) {
8162 case FCmpInst::FCMP_OLT:
8163 case FCmpInst::FCMP_UGE:
8164 // sqrt(X) < 0.0 --> false
8165 // sqrt(X) u>= 0.0 --> true
8166 llvm_unreachable("fcmp should have simplified");
8167 case FCmpInst::FCMP_ULT:
8168 case FCmpInst::FCMP_ULE:
8169 case FCmpInst::FCMP_OGT:
8170 case FCmpInst::FCMP_OGE:
8171 case FCmpInst::FCMP_OEQ:
8172 case FCmpInst::FCMP_UNE:
8173 // sqrt(X) u< 0.0 --> X u< 0.0
8174 // sqrt(X) u<= 0.0 --> X u<= 0.0
8175 // sqrt(X) > 0.0 --> X > 0.0
8176 // sqrt(X) >= 0.0 --> X >= 0.0
8177 // sqrt(X) == 0.0 --> X == 0.0
8178 // sqrt(X) u!= 0.0 --> X u!= 0.0
8179 return IC.replaceOperand(I, 0, X);
8180
8181 case FCmpInst::FCMP_OLE:
8182 // sqrt(X) <= 0.0 --> X == 0.0
8183 return ReplacePredAndOp0(FCmpInst::FCMP_OEQ);
8184 case FCmpInst::FCMP_UGT:
8185 // sqrt(X) u> 0.0 --> X u!= 0.0
8186 return ReplacePredAndOp0(FCmpInst::FCMP_UNE);
8187 case FCmpInst::FCMP_UEQ:
8188 // sqrt(X) u== 0.0 --> X u<= 0.0
8189 return ReplacePredAndOp0(FCmpInst::FCMP_ULE);
8190 case FCmpInst::FCMP_ONE:
8191 // sqrt(X) != 0.0 --> X > 0.0
8192 return ReplacePredAndOp0(FCmpInst::FCMP_OGT);
8193 case FCmpInst::FCMP_ORD:
8194 // !isnan(sqrt(X)) --> X >= 0.0
8195 return ReplacePredAndOp0(FCmpInst::FCMP_OGE);
8196 case FCmpInst::FCMP_UNO:
8197 // isnan(sqrt(X)) --> X u< 0.0
8198 return ReplacePredAndOp0(FCmpInst::FCMP_ULT);
8199 default:
8200 llvm_unreachable("Unexpected predicate!");
8201 }
8202}
8203
8205 CmpInst::Predicate Pred = I.getPredicate();
8206 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
8207
8208 // Canonicalize fneg as Op1.
8209 if (match(Op0, m_FNeg(m_Value())) && !match(Op1, m_FNeg(m_Value()))) {
8210 std::swap(Op0, Op1);
8211 Pred = I.getSwappedPredicate();
8212 }
8213
8214 if (!match(Op1, m_FNeg(m_Specific(Op0))))
8215 return nullptr;
8216
8217 // Replace the negated operand with 0.0:
8218 // fcmp Pred Op0, -Op0 --> fcmp Pred Op0, 0.0
8219 Constant *Zero = ConstantFP::getZero(Op0->getType());
8220 return new FCmpInst(Pred, Op0, Zero, "", &I);
8221}
8222
8224 Constant *RHSC, InstCombinerImpl &CI) {
8225 const CmpInst::Predicate Pred = I.getPredicate();
8226 Value *X = LHSI->getOperand(0);
8227 Value *Y = LHSI->getOperand(1);
8228 switch (Pred) {
8229 default:
8230 break;
8231 case FCmpInst::FCMP_UGT:
8232 case FCmpInst::FCMP_ULT:
8233 case FCmpInst::FCMP_UNE:
8234 case FCmpInst::FCMP_OEQ:
8235 case FCmpInst::FCMP_OGE:
8236 case FCmpInst::FCMP_OLE:
8237 // The optimization is not valid if X and Y are infinities of the same
8238 // sign, i.e. the inf - inf = nan case. If the fsub has the ninf or nnan
8239 // flag then we can assume we do not have that case. Otherwise we might be
8240 // able to prove that either X or Y is not infinity.
8241 if (!LHSI->hasNoNaNs() && !LHSI->hasNoInfs() &&
8242 !isKnownNeverInfinity(Y, /*Depth=*/0,
8244 !isKnownNeverInfinity(X, /*Depth=*/0,
8246 break;
8247
8248 [[fallthrough]];
8249 case FCmpInst::FCMP_OGT:
8250 case FCmpInst::FCMP_OLT:
8251 case FCmpInst::FCMP_ONE:
8252 case FCmpInst::FCMP_UEQ:
8253 case FCmpInst::FCMP_UGE:
8254 case FCmpInst::FCMP_ULE:
8255 // fcmp pred (x - y), 0 --> fcmp pred x, y
8256 if (match(RHSC, m_AnyZeroFP()) &&
8257 I.getFunction()->getDenormalMode(
8258 LHSI->getType()->getScalarType()->getFltSemantics()) ==
8260 CI.replaceOperand(I, 0, X);
8261 CI.replaceOperand(I, 1, Y);
8262 return &I;
8263 }
8264 break;
8265 }
8266
8267 return nullptr;
8268}
8269
8271 InstCombinerImpl &IC) {
8272 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
8273 Type *OpType = LHS->getType();
8274 CmpInst::Predicate Pred = I.getPredicate();
8275
8276 bool FloorX = match(LHS, m_Intrinsic<Intrinsic::floor>(m_Specific(RHS)));
8277 bool CeilX = match(LHS, m_Intrinsic<Intrinsic::ceil>(m_Specific(RHS)));
8278
8279 if (!FloorX && !CeilX) {
8280 if ((FloorX = match(RHS, m_Intrinsic<Intrinsic::floor>(m_Specific(LHS)))) ||
8281 (CeilX = match(RHS, m_Intrinsic<Intrinsic::ceil>(m_Specific(LHS))))) {
8282 std::swap(LHS, RHS);
8283 Pred = I.getSwappedPredicate();
8284 }
8285 }
8286
8287 switch (Pred) {
8288 case FCmpInst::FCMP_OLE:
8289 // fcmp ole floor(x), x => fcmp ord x, 0
8290 if (FloorX)
8292 "", &I);
8293 break;
8294 case FCmpInst::FCMP_OGT:
8295 // fcmp ogt floor(x), x => false
8296 if (FloorX)
8297 return IC.replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8298 break;
8299 case FCmpInst::FCMP_OGE:
8300 // fcmp oge ceil(x), x => fcmp ord x, 0
8301 if (CeilX)
8303 "", &I);
8304 break;
8305 case FCmpInst::FCMP_OLT:
8306 // fcmp olt ceil(x), x => false
8307 if (CeilX)
8308 return IC.replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8309 break;
8310 case FCmpInst::FCMP_ULE:
8311 // fcmp ule floor(x), x => true
8312 if (FloorX)
8313 return IC.replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8314 break;
8315 case FCmpInst::FCMP_UGT:
8316 // fcmp ugt floor(x), x => fcmp uno x, 0
8317 if (FloorX)
8319 "", &I);
8320 break;
8321 case FCmpInst::FCMP_UGE:
8322 // fcmp uge ceil(x), x => true
8323 if (CeilX)
8324 return IC.replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8325 break;
8326 case FCmpInst::FCMP_ULT:
8327 // fcmp ult ceil(x), x => fcmp uno x, 0
8328 if (CeilX)
8330 "", &I);
8331 break;
8332 default:
8333 break;
8334 }
8335
8336 return nullptr;
8337}
8338
8340 bool Changed = false;
8341
8342 /// Orders the operands of the compare so that they are listed from most
8343 /// complex to least complex. This puts constants before unary operators,
8344 /// before binary operators.
8345 if (getComplexity(I.getOperand(0)) < getComplexity(I.getOperand(1))) {
8346 I.swapOperands();
8347 Changed = true;
8348 }
8349
8350 const CmpInst::Predicate Pred = I.getPredicate();
8351 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
8352 if (Value *V = simplifyFCmpInst(Pred, Op0, Op1, I.getFastMathFlags(),
8354 return replaceInstUsesWith(I, V);
8355
8356 // Simplify 'fcmp pred X, X'
8357 Type *OpType = Op0->getType();
8358 assert(OpType == Op1->getType() && "fcmp with different-typed operands?");
8359 if (Op0 == Op1) {
8360 switch (Pred) {
8361 default: break;
8362 case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y)
8363 case FCmpInst::FCMP_ULT: // True if unordered or less than
8364 case FCmpInst::FCMP_UGT: // True if unordered or greater than
8365 case FCmpInst::FCMP_UNE: // True if unordered or not equal
8366 // Canonicalize these to be 'fcmp uno %X, 0.0'.
8367 I.setPredicate(FCmpInst::FCMP_UNO);
8368 I.setOperand(1, Constant::getNullValue(OpType));
8369 return &I;
8370
8371 case FCmpInst::FCMP_ORD: // True if ordered (no nans)
8372 case FCmpInst::FCMP_OEQ: // True if ordered and equal
8373 case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal
8374 case FCmpInst::FCMP_OLE: // True if ordered and less than or equal
8375 // Canonicalize these to be 'fcmp ord %X, 0.0'.
8376 I.setPredicate(FCmpInst::FCMP_ORD);
8377 I.setOperand(1, Constant::getNullValue(OpType));
8378 return &I;
8379 }
8380 }
8381
8382 if (I.isCommutative()) {
8383 if (auto Pair = matchSymmetricPair(I.getOperand(0), I.getOperand(1))) {
8384 replaceOperand(I, 0, Pair->first);
8385 replaceOperand(I, 1, Pair->second);
8386 return &I;
8387 }
8388 }
8389
8390 // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand,
8391 // then canonicalize the operand to 0.0.
8392 if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
8393 if (!match(Op0, m_PosZeroFP()) &&
8394 isKnownNeverNaN(Op0, 0, getSimplifyQuery().getWithInstruction(&I)))
8395 return replaceOperand(I, 0, ConstantFP::getZero(OpType));
8396
8397 if (!match(Op1, m_PosZeroFP()) &&
8398 isKnownNeverNaN(Op1, 0, getSimplifyQuery().getWithInstruction(&I)))
8399 return replaceOperand(I, 1, ConstantFP::getZero(OpType));
8400 }
8401
8402 // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y
8403 Value *X, *Y;
8404 if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y))))
8405 return new FCmpInst(I.getSwappedPredicate(), X, Y, "", &I);
8406
8408 return R;
8409
8410 // Test if the FCmpInst instruction is used exclusively by a select as
8411 // part of a minimum or maximum operation. If so, refrain from doing
8412 // any other folding. This helps out other analyses which understand
8413 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
8414 // and CodeGen. And in this case, at least one of the comparison
8415 // operands has at least one user besides the compare (the select),
8416 // which would often largely negate the benefit of folding anyway.
8417 if (I.hasOneUse())
8418 if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) {
8419 Value *A, *B;
8421 if (SPR.Flavor != SPF_UNKNOWN)
8422 return nullptr;
8423 }
8424
8425 // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0:
8426 // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0
8427 if (match(Op1, m_AnyZeroFP()) && !match(Op1, m_PosZeroFP()))
8428 return replaceOperand(I, 1, ConstantFP::getZero(OpType));
8429
8430 // Canonicalize:
8431 // fcmp olt X, +inf -> fcmp one X, +inf
8432 // fcmp ole X, +inf -> fcmp ord X, 0
8433 // fcmp ogt X, +inf -> false
8434 // fcmp oge X, +inf -> fcmp oeq X, +inf
8435 // fcmp ult X, +inf -> fcmp une X, +inf
8436 // fcmp ule X, +inf -> true
8437 // fcmp ugt X, +inf -> fcmp uno X, 0
8438 // fcmp uge X, +inf -> fcmp ueq X, +inf
8439 // fcmp olt X, -inf -> false
8440 // fcmp ole X, -inf -> fcmp oeq X, -inf
8441 // fcmp ogt X, -inf -> fcmp one X, -inf
8442 // fcmp oge X, -inf -> fcmp ord X, 0
8443 // fcmp ult X, -inf -> fcmp uno X, 0
8444 // fcmp ule X, -inf -> fcmp ueq X, -inf
8445 // fcmp ugt X, -inf -> fcmp une X, -inf
8446 // fcmp uge X, -inf -> true
8447 const APFloat *C;
8448 if (match(Op1, m_APFloat(C)) && C->isInfinity()) {
8449 switch (C->isNegative() ? FCmpInst::getSwappedPredicate(Pred) : Pred) {
8450 default:
8451 break;
8452 case FCmpInst::FCMP_ORD:
8453 case FCmpInst::FCMP_UNO:
8456 case FCmpInst::FCMP_OGT:
8457 case FCmpInst::FCMP_ULE:
8458 llvm_unreachable("Should be simplified by InstSimplify");
8459 case FCmpInst::FCMP_OLT:
8460 return new FCmpInst(FCmpInst::FCMP_ONE, Op0, Op1, "", &I);
8461 case FCmpInst::FCMP_OLE:
8462 return new FCmpInst(FCmpInst::FCMP_ORD, Op0, ConstantFP::getZero(OpType),
8463 "", &I);
8464 case FCmpInst::FCMP_OGE:
8465 return new FCmpInst(FCmpInst::FCMP_OEQ, Op0, Op1, "", &I);
8466 case FCmpInst::FCMP_ULT:
8467 return new FCmpInst(FCmpInst::FCMP_UNE, Op0, Op1, "", &I);
8468 case FCmpInst::FCMP_UGT:
8469 return new FCmpInst(FCmpInst::FCMP_UNO, Op0, ConstantFP::getZero(OpType),
8470 "", &I);
8471 case FCmpInst::FCMP_UGE:
8472 return new FCmpInst(FCmpInst::FCMP_UEQ, Op0, Op1, "", &I);
8473 }
8474 }
8475
8476 // Ignore signbit of bitcasted int when comparing equality to FP 0.0:
8477 // fcmp oeq/une (bitcast X), 0.0 --> (and X, SignMaskC) ==/!= 0
8478 if (match(Op1, m_PosZeroFP()) &&
8481 if (Pred == FCmpInst::FCMP_OEQ)
8482 IntPred = ICmpInst::ICMP_EQ;
8483 else if (Pred == FCmpInst::FCMP_UNE)
8484 IntPred = ICmpInst::ICMP_NE;
8485
8486 if (IntPred != ICmpInst::BAD_ICMP_PREDICATE) {
8487 Type *IntTy = X->getType();
8488 const APInt &SignMask = ~APInt::getSignMask(IntTy->getScalarSizeInBits());
8489 Value *MaskX = Builder.CreateAnd(X, ConstantInt::get(IntTy, SignMask));
8490 return new ICmpInst(IntPred, MaskX, ConstantInt::getNullValue(IntTy));
8491 }
8492 }
8493
8494 // Handle fcmp with instruction LHS and constant RHS.
8495 Instruction *LHSI;
8496 Constant *RHSC;
8497 if (match(Op0, m_Instruction(LHSI)) && match(Op1, m_Constant(RHSC))) {
8498 switch (LHSI->getOpcode()) {
8499 case Instruction::Select:
8500 // fcmp eq (cond ? x : -x), 0 --> fcmp eq x, 0
8501 if (FCmpInst::isEquality(Pred) && match(RHSC, m_AnyZeroFP()) &&
8503 return replaceOperand(I, 0, X);
8504 if (Instruction *NV = FoldOpIntoSelect(I, cast<SelectInst>(LHSI)))
8505 return NV;
8506 break;
8507 case Instruction::FSub:
8508 if (LHSI->hasOneUse())
8509 if (Instruction *NV = foldFCmpFSubIntoFCmp(I, LHSI, RHSC, *this))
8510 return NV;
8511 break;
8512 case Instruction::PHI:
8513 if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI)))
8514 return NV;
8515 break;
8516 case Instruction::SIToFP:
8517 case Instruction::UIToFP:
8518 if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC))
8519 return NV;
8520 break;
8521 case Instruction::FDiv:
8522 if (Instruction *NV = foldFCmpReciprocalAndZero(I, LHSI, RHSC))
8523 return NV;
8524 break;
8525 case Instruction::Load:
8526 if (auto *GEP = dyn_cast<GetElementPtrInst>(LHSI->getOperand(0)))
8527 if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)))
8529 cast<LoadInst>(LHSI), GEP, GV, I))
8530 return Res;
8531 break;
8532 }
8533 }
8534
8535 if (Instruction *R = foldFabsWithFcmpZero(I, *this))
8536 return R;
8537
8538 if (Instruction *R = foldSqrtWithFcmpZero(I, *this))
8539 return R;
8540
8541 if (Instruction *R = foldFCmpWithFloorAndCeil(I, *this))
8542 return R;
8543
8544 if (match(Op0, m_FNeg(m_Value(X)))) {
8545 // fcmp pred (fneg X), C --> fcmp swap(pred) X, -C
8546 Constant *C;
8547 if (match(Op1, m_Constant(C)))
8548 if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL))
8549 return new FCmpInst(I.getSwappedPredicate(), X, NegC, "", &I);
8550 }
8551
8552 // fcmp (fadd X, 0.0), Y --> fcmp X, Y
8553 if (match(Op0, m_FAdd(m_Value(X), m_AnyZeroFP())))
8554 return new FCmpInst(Pred, X, Op1, "", &I);
8555
8556 // fcmp X, (fadd Y, 0.0) --> fcmp X, Y
8557 if (match(Op1, m_FAdd(m_Value(Y), m_AnyZeroFP())))
8558 return new FCmpInst(Pred, Op0, Y, "", &I);
8559
8560 if (match(Op0, m_FPExt(m_Value(X)))) {
8561 // fcmp (fpext X), (fpext Y) -> fcmp X, Y
8562 if (match(Op1, m_FPExt(m_Value(Y))) && X->getType() == Y->getType())
8563 return new FCmpInst(Pred, X, Y, "", &I);
8564
8565 const APFloat *C;
8566 if (match(Op1, m_APFloat(C))) {
8567 const fltSemantics &FPSem =
8568 X->getType()->getScalarType()->getFltSemantics();
8569 bool Lossy;
8570 APFloat TruncC = *C;
8571 TruncC.convert(FPSem, APFloat::rmNearestTiesToEven, &Lossy);
8572
8573 if (Lossy) {
8574 // X can't possibly equal the higher-precision constant, so reduce any
8575 // equality comparison.
8576 // TODO: Other predicates can be handled via getFCmpCode().
8577 switch (Pred) {
8578 case FCmpInst::FCMP_OEQ:
8579 // X is ordered and equal to an impossible constant --> false
8580 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
8581 case FCmpInst::FCMP_ONE:
8582 // X is ordered and not equal to an impossible constant --> ordered
8583 return new FCmpInst(FCmpInst::FCMP_ORD, X,
8584 ConstantFP::getZero(X->getType()));
8585 case FCmpInst::FCMP_UEQ:
8586 // X is unordered or equal to an impossible constant --> unordered
8587 return new FCmpInst(FCmpInst::FCMP_UNO, X,
8588 ConstantFP::getZero(X->getType()));
8589 case FCmpInst::FCMP_UNE:
8590 // X is unordered or not equal to an impossible constant --> true
8591 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
8592 default:
8593 break;
8594 }
8595 }
8596
8597 // fcmp (fpext X), C -> fcmp X, (fptrunc C) if fptrunc is lossless
8598 // Avoid lossy conversions and denormals.
8599 // Zero is a special case that's OK to convert.
8600 APFloat Fabs = TruncC;
8601 Fabs.clearSign();
8602 if (!Lossy &&
8603 (Fabs.isZero() || !(Fabs < APFloat::getSmallestNormalized(FPSem)))) {
8604 Constant *NewC = ConstantFP::get(X->getType(), TruncC);
8605 return new FCmpInst(Pred, X, NewC, "", &I);
8606 }
8607 }
8608 }
8609
8610 // Convert a sign-bit test of an FP value into a cast and integer compare.
8611 // TODO: Simplify if the copysign constant is 0.0 or NaN.
8612 // TODO: Handle non-zero compare constants.
8613 // TODO: Handle other predicates.
8614 if (match(Op0, m_OneUse(m_Intrinsic<Intrinsic::copysign>(m_APFloat(C),
8615 m_Value(X)))) &&
8616 match(Op1, m_AnyZeroFP()) && !C->isZero() && !C->isNaN()) {
8617 Type *IntType = Builder.getIntNTy(X->getType()->getScalarSizeInBits());
8618 if (auto *VecTy = dyn_cast<VectorType>(OpType))
8619 IntType = VectorType::get(IntType, VecTy->getElementCount());
8620
8621 // copysign(non-zero constant, X) < 0.0 --> (bitcast X) < 0
8622 if (Pred == FCmpInst::FCMP_OLT) {
8623 Value *IntX = Builder.CreateBitCast(X, IntType);
8624 return new ICmpInst(ICmpInst::ICMP_SLT, IntX,
8625 ConstantInt::getNullValue(IntType));
8626 }
8627 }
8628
8629 {
8630 Value *CanonLHS = nullptr, *CanonRHS = nullptr;
8631 match(Op0, m_Intrinsic<Intrinsic::canonicalize>(m_Value(CanonLHS)));
8632 match(Op1, m_Intrinsic<Intrinsic::canonicalize>(m_Value(CanonRHS)));
8633
8634 // (canonicalize(x) == x) => (x == x)
8635 if (CanonLHS == Op1)
8636 return new FCmpInst(Pred, Op1, Op1, "", &I);
8637
8638 // (x == canonicalize(x)) => (x == x)
8639 if (CanonRHS == Op0)
8640 return new FCmpInst(Pred, Op0, Op0, "", &I);
8641
8642 // (canonicalize(x) == canonicalize(y)) => (x == y)
8643 if (CanonLHS && CanonRHS)
8644 return new FCmpInst(Pred, CanonLHS, CanonRHS, "", &I);
8645 }
8646
8647 if (I.getType()->isVectorTy())
8648 if (Instruction *Res = foldVectorCmp(I, Builder))
8649 return Res;
8650
8651 return Changed ? &I : nullptr;
8652}
AMDGPU Register Bank Select
Rewrite undef for PHI
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
#define Check(C,...)
Hexagon Common GEP
static Instruction * foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI, Constant *RHSC)
Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary.
static Instruction * foldFabsWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC)
Optimize fabs(X) compared with zero.
static Value * rewriteGEPAsOffset(Value *Start, Value *Base, GEPNoWrapFlags NW, const DataLayout &DL, SetVector< Value * > &Explored, InstCombiner &IC)
Returns a re-written value of Start as an indexed GEP using Base as a pointer.
static bool addWithOverflow(APInt &Result, const APInt &In1, const APInt &In2, bool IsSigned=false)
Compute Result = In1+In2, returning true if the result overflowed for this type.
static Instruction * foldICmpAndXX(ICmpInst &I, const SimplifyQuery &Q, InstCombinerImpl &IC)
static Instruction * foldVectorCmp(CmpInst &Cmp, InstCombiner::BuilderTy &Builder)
static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q, unsigned Depth=0)
static Value * createLogicFromTable(const std::bitset< 4 > &Table, Value *Op0, Value *Op1, IRBuilderBase &Builder, bool HasOneUse)
static Instruction * foldICmpOfUAddOv(ICmpInst &I)
static bool isChainSelectCmpBranch(const SelectInst *SI)
Return true when the instruction sequence within a block is select-cmp-br.
static Instruction * foldICmpInvariantGroup(ICmpInst &I)
static Instruction * foldReductionIdiom(ICmpInst &I, InstCombiner::BuilderTy &Builder, const DataLayout &DL)
This function folds patterns produced by lowering of reduce idioms, such as llvm.vector....
static Instruction * canonicalizeICmpBool(ICmpInst &I, InstCombiner::BuilderTy &Builder)
Integer compare with boolean values can always be turned into bitwise ops.
static Instruction * foldFCmpFSubIntoFCmp(FCmpInst &I, Instruction *LHSI, Constant *RHSC, InstCombinerImpl &CI)
static Value * foldICmpOrXorSubChain(ICmpInst &Cmp, BinaryOperator *Or, InstCombiner::BuilderTy &Builder)
Fold icmp eq/ne (or (xor/sub (X1, X2), xor/sub (X3, X4))), 0.
static bool hasBranchUse(ICmpInst &I)
Given an icmp instruction, return true if any use of this comparison is a branch on sign bit comparis...
static Value * foldICmpWithLowBitMaskedVal(CmpPredicate Pred, Value *Op0, Value *Op1, const SimplifyQuery &Q, InstCombiner &IC)
Some comparisons can be simplified.
static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth)
When performing a comparison against a constant, it is possible that not all the bits in the LHS are ...
static Instruction * foldICmpShlLHSC(ICmpInst &Cmp, Instruction *Shl, const APInt &C)
Fold icmp (shl nuw C2, Y), C.
static Instruction * foldFCmpWithFloorAndCeil(FCmpInst &I, InstCombinerImpl &IC)
static Instruction * foldICmpXorXX(ICmpInst &I, const SimplifyQuery &Q, InstCombinerImpl &IC)
static Instruction * foldICmpOfCmpIntrinsicWithConstant(CmpPredicate Pred, IntrinsicInst *I, const APInt &C, InstCombiner::BuilderTy &Builder)
static Instruction * processUMulZExtIdiom(ICmpInst &I, Value *MulVal, const APInt *OtherVal, InstCombinerImpl &IC)
Recognize and process idiom involving test for multiplication overflow.
static Instruction * foldSqrtWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC)
Optimize sqrt(X) compared with zero.
static Instruction * foldFCmpFNegCommonOp(FCmpInst &I)
static Instruction * foldICmpWithHighBitMask(ICmpInst &Cmp, InstCombiner::BuilderTy &Builder)
static ICmpInst * canonicalizeCmpWithConstant(ICmpInst &I)
If we have an icmp le or icmp ge instruction with a constant operand, turn it into the appropriate ic...
static Instruction * foldICmpIntrinsicWithIntrinsic(ICmpInst &Cmp, InstCombiner::BuilderTy &Builder)
Fold an icmp with LLVM intrinsics.
static Instruction * foldICmpUSubSatOrUAddSatWithConstant(CmpPredicate Pred, SaturatingInst *II, const APInt &C, InstCombiner::BuilderTy &Builder)
static Instruction * foldICmpPow2Test(ICmpInst &I, InstCombiner::BuilderTy &Builder)
static bool subWithOverflow(APInt &Result, const APInt &In1, const APInt &In2, bool IsSigned=false)
Compute Result = In1-In2, returning true if the result overflowed for this type.
static bool canRewriteGEPAsOffset(Value *Start, Value *Base, GEPNoWrapFlags &NW, const DataLayout &DL, SetVector< Value * > &Explored)
Returns true if we can rewrite Start as a GEP with pointer Base and some integer offset.
static Instruction * foldICmpXNegX(ICmpInst &I, InstCombiner::BuilderTy &Builder)
static Instruction * processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B, ConstantInt *CI2, ConstantInt *CI1, InstCombinerImpl &IC)
The caller has matched a pattern of the form: I = icmp ugt (add (add A, B), CI2), CI1 If this is of t...
static Value * foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ, InstCombiner::BuilderTy &Builder)
static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C)
Returns true if the exploded icmp can be expressed as a signed comparison to zero and updates the pre...
static Instruction * transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS, CmpPredicate Cond, const DataLayout &DL, InstCombiner &IC)
Converts (CMP GEPLHS, RHS) if this change would make RHS a constant.
static Instruction * foldCtpopPow2Test(ICmpInst &I, IntrinsicInst *CtpopLhs, const APInt &CRhs, InstCombiner::BuilderTy &Builder, const SimplifyQuery &Q)
static void setInsertionPoint(IRBuilder<> &Builder, Value *V, bool Before=true)
static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS, bool IsSigned)
static Value * foldICmpWithTruncSignExtendedVal(ICmpInst &I, InstCombiner::BuilderTy &Builder)
Some comparisons can be simplified.
static Instruction * foldICmpOrXX(ICmpInst &I, const SimplifyQuery &Q, InstCombinerImpl &IC)
This file provides internal interfaces used to implement the InstCombine.
This file provides the interface for the instcombine pass implementation.
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:533
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
mir Rename Register Operands
#define T1
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:166
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:39
Value * RHS
Value * LHS
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition: APFloat.cpp:5465
void clearSign()
Definition: APFloat.h:1295
bool isZero() const
Definition: APFloat.h:1436
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
Definition: APFloat.h:1155
APInt bitcastToAPInt() const
Definition: APFloat.h:1346
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition: APFloat.h:1135
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition: APFloat.h:1095
FPClassTest classify() const
Return the FPClassTest which will return true for the value.
Definition: APFloat.cpp:5452
opStatus roundToIntegral(roundingMode RM)
Definition: APFloat.h:1245
Class for arbitrary precision integers.
Definition: APInt.h:78
APInt udiv(const APInt &RHS) const
Unsigned division operation.
Definition: APInt.cpp:1547
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:234
static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
Definition: APInt.cpp:1732
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
Definition: APInt.h:449
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:986
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:229
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition: APInt.h:423
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1520
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1492
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:910
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:206
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition: APInt.h:1330
APInt abs() const
Get the absolute value.
Definition: APInt.h:1773
unsigned ceilLogBase2() const
Definition: APInt.h:1742
bool sgt(const APInt &RHS) const
Signed greater than comparison.
Definition: APInt.h:1201
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition: APInt.h:371
APInt usub_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1922
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition: APInt.h:1182
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition: APInt.h:380
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
Definition: APInt.h:466
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1468
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.h:1111
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:209
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
Definition: APInt.h:216
bool isNegative() const
Determine sign of this APInt.
Definition: APInt.h:329
APInt sadd_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1902
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1249
bool eq(const APInt &RHS) const
Equality comparison.
Definition: APInt.h:1079
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
Definition: APInt.cpp:1618
APInt uadd_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1909
void negate()
Negate this APInt in place.
Definition: APInt.h:1450
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition: APInt.h:1618
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition: APInt.h:1577
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:219
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Definition: APInt.h:356
unsigned countl_one() const
Count the number of leading one bits.
Definition: APInt.h:1594
unsigned logBase2() const
Definition: APInt.h:1739
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition: APInt.h:475
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition: APInt.h:827
bool isMask(unsigned numBits) const
Definition: APInt.h:488
bool isMaxSignedValue() const
Determine if this is the largest signed value.
Definition: APInt.h:405
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition: APInt.h:1150
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition: APInt.h:873
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:440
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:306
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition: APInt.h:296
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition: APInt.h:200
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
Definition: APInt.h:1237
APInt ssub_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1915
bool isOne() const
Determine if this is a value of 1.
Definition: APInt.h:389
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition: APInt.h:286
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:239
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:851
unsigned countr_one() const
Count the number of trailing one bits.
Definition: APInt.h:1635
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition: APInt.h:1221
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
an instruction to allocate memory on the stack
Definition: Instructions.h:63
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
Class to represent array types.
Definition: DerivedTypes.h:395
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:416
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:459
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:239
BinaryOps getOpcode() const
Definition: InstrTypes.h:370
static BinaryOperator * CreateNot(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
Conditional or Unconditional Branch instruction.
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1294
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:661
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Definition: InstrTypes.h:988
Predicate getStrictPredicate() const
For example, SGE -> SGT, SLE -> SLT, ULE -> ULT, UGE -> UGT.
Definition: InstrTypes.h:856
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:673
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:676
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:690
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:702
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:703
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:679
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:688
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:677
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:678
@ ICMP_UGE
unsigned greater or equal
Definition: InstrTypes.h:697
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:696
@ ICMP_SGT
signed greater than
Definition: InstrTypes.h:700
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:687
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:681
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:684
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:698
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:685
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:680
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:682
@ ICMP_EQ
equal
Definition: InstrTypes.h:694
@ ICMP_NE
not equal
Definition: InstrTypes.h:695
@ ICMP_SGE
signed greater or equal
Definition: InstrTypes.h:701
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:689
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:699
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:686
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:675
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:683
bool isSigned() const
Definition: InstrTypes.h:928
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition: InstrTypes.h:825
bool isTrueWhenEqual() const
This is just a convenience.
Definition: InstrTypes.h:940
static CmpInst * Create(OtherOps Op, Predicate Pred, Value *S1, Value *S2, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Construct a compare instruction, given the opcode, the predicate and the two operands.
Predicate getNonStrictPredicate() const
For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
Definition: InstrTypes.h:869
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition: InstrTypes.h:787
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:763
bool isStrictPredicate() const
Definition: InstrTypes.h:841
Predicate getFlippedStrictnessPredicate() const
For predicate of kind "is X or equal to 0" returns the predicate "is X".
Definition: InstrTypes.h:891
bool isIntPredicate() const
Definition: InstrTypes.h:781
bool isUnsigned() const
Definition: InstrTypes.h:934
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
Definition: CmpPredicate.h:22
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2307
static Constant * getPointerBitCastOrAddrSpaceCast(Constant *C, Type *Ty)
Create a BitCast or AddrSpaceCast for a pointer type depending on the address space.
Definition: Constants.cpp:2268
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2644
static Constant * getNot(Constant *C)
Definition: Constants.cpp:2631
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2293
static Constant * getXor(Constant *C1, Constant *C2)
Definition: Constants.cpp:2658
static Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2637
static Constant * getNeg(Constant *C, bool HasNSW=false)
Definition: Constants.cpp:2625
static Constant * getZero(Type *Ty, bool Negative=false)
Definition: Constants.cpp:1057
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
Definition: Constants.h:258
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:866
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:208
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
Definition: Constants.h:126
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:873
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
Definition: Constants.h:151
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:148
static ConstantInt * getBool(LLVMContext &Context, bool V)
Definition: Constants.cpp:880
This class represents a range of values.
Definition: ConstantRange.h:47
ConstantRange add(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an addition of a value in this ran...
std::optional< ConstantRange > exactUnionWith(const ConstantRange &CR) const
Union the two ranges and return the result if it can be represented exactly, otherwise return std::nu...
bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const
Set up Pred and RHS such that ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.
ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
const APInt * getSingleElement() const
If this set contains a single element, return it, otherwise return null.
ConstantRange truncate(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly smaller than the current typ...
ConstantRange difference(const ConstantRange &CR) const
Subtract the specified range from this range (aka relative complement of the sets).
bool isEmptySet() const
Return true if this set contains no members.
static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
ConstantRange inverse() const
Return a new range that is the logical not of the current set.
std::optional< ConstantRange > exactIntersectWith(const ConstantRange &CR) const
Intersect the two ranges and return the result if it can be represented exactly, otherwise return std...
ConstantRange intersectWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the intersection of this range with another range.
ConstantRange sub(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a subtraction of a value in this r...
static ConstantRange makeExactNoWrapRegion(Instruction::BinaryOps BinOp, const APInt &Other, unsigned NoWrapKind)
Produce the range that contains X if and only if "X BinOp Other" does not wrap.
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
Definition: Constants.cpp:1472
This is an important base class in LLVM.
Definition: Constant.h:42
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
Definition: Constants.cpp:403
static Constant * replaceUndefsWith(Constant *C, Constant *Replacement)
Try to replace undefined constant C or undefined elements in C with Replacement.
Definition: Constants.cpp:784
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:420
const APInt & getUniqueInteger() const
If C is a constant integer then return its value, otherwise C must be a vector of constant integers,...
Definition: Constants.cpp:1771
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:373
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition: Constants.cpp:90
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Definition: DataLayout.cpp:851
unsigned getPointerTypeSizeInBits(Type *) const
Layout pointer size, in bits, based on the type.
Definition: DataLayout.cpp:743
IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
Definition: DataLayout.cpp:878
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:457
Type * getSmallestLegalIntType(LLVMContext &C, unsigned Width=0) const
Returns the smallest integer type with size at least as big as Width bits.
Definition: DataLayout.cpp:866
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:156
iterator end()
Definition: DenseMap.h:84
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition: DenseMap.h:147
ArrayRef< BranchInst * > conditionsFor(const Value *V) const
Access the list of branches which affect this value.
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Definition: Dominators.cpp:122
static ExtractValueInst * Create(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This instruction compares its operands according to the predicate given to the constructor.
bool isEquality() const
Represents flags for the getelementptr instruction/expression.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
GEPNoWrapFlags intersectForOffsetAdd(GEPNoWrapFlags Other) const
Given (gep (gep p, x), y), determine the nowrap flags for (gep p, x+y).
static GEPNoWrapFlags none()
bool isInBounds() const
Test whether this is an inbounds GEP, as defined by LangRef.html.
Definition: Operator.h:435
Type * getSourceElementType() const
Definition: Operator.cpp:70
Value * getPointerOperand()
Definition: Operator.h:462
GEPNoWrapFlags getNoWrapFlags() const
Definition: Operator.h:430
bool hasAllConstantIndices() const
Return true if all of the indices of this GEP are constant integers.
Definition: Operator.h:509
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:933
Type * getValueType() const
Definition: GlobalValue.h:296
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool isConstant() const
If the value is a global constant, its value is immutable throughout the runtime execution of the pro...
bool hasDefinitiveInitializer() const
hasDefinitiveInitializer - Whether the global variable has an initializer, and any other instances of...
This instruction compares its operands according to the predicate given to the constructor.
static bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
Predicate getFlippedSignednessPredicate() const
For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->EQ.
Predicate getSignedPredicate() const
For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
bool isEquality() const
Return true if this predicate is either EQ or NE.
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
bool isRelational() const
Return true if the predicate is relational (not EQ or NE).
Predicate getUnsignedPredicate() const
For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:108
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2280
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2493
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Definition: IRBuilder.h:553
Value * CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2288
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1153
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition: IRBuilder.h:2549
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition: IRBuilder.h:480
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1474
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})
Definition: IRBuilder.h:2180
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2268
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Definition: IRBuilder.h:1868
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
Definition: IRBuilder.h:1727
Value * createIsFPClass(Value *FPNum, unsigned Test)
Definition: IRBuilder.cpp:1238
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Definition: IRBuilder.cpp:890
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:500
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2398
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2429
Value * CreateNot(Value *V, const Twine &Name="")
Definition: IRBuilder.h:1751
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2264
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1381
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2146
Value * CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2272
CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
Definition: IRBuilder.cpp:871
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1453
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Definition: IRBuilder.h:2027
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1512
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1364
ConstantInt * getFalse()
Get the constant value for i1 false.
Definition: IRBuilder.h:485
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2443
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Definition: IRBuilder.h:2013
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1534
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1665
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2296
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:2219
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
Definition: IRBuilder.h:2577
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition: IRBuilder.h:194
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1556
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2374
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Definition: IRBuilder.h:530
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
Definition: IRBuilder.h:516
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1441
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1398
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2699
Instruction * foldICmpShrConstant(ICmpInst &Cmp, BinaryOperator *Shr, const APInt &C)
Fold icmp ({al}shr X, Y), C.
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false)
Given an instruction with a select as one operand and a constant as the other operand,...
Instruction * foldICmpWithZextOrSext(ICmpInst &ICmp)
Instruction * foldICmpSelectConstant(ICmpInst &Cmp, SelectInst *Select, ConstantInt *C)
Instruction * foldICmpSRemConstant(ICmpInst &Cmp, BinaryOperator *UDiv, const APInt &C)
Instruction * foldICmpBinOpWithConstant(ICmpInst &Cmp, BinaryOperator *BO, const APInt &C)
Fold an icmp with BinaryOp and constant operand: icmp Pred BO, C.
Instruction * foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or, const APInt &C)
Fold icmp (or X, Y), C.
Instruction * foldICmpTruncWithTruncOrExt(ICmpInst &Cmp, const SimplifyQuery &Q)
Fold icmp (trunc nuw/nsw X), (trunc nuw/nsw Y).
Instruction * foldSignBitTest(ICmpInst &I)
Fold equality-comparison between zero and any (maybe truncated) right-shift by one-less-than-bitwidth...
Instruction * foldOpIntoPhi(Instruction &I, PHINode *PN, bool AllowMultipleUses=false)
Given a binary operator, cast instruction, or select which has a PHI node as operand #0,...
Value * insertRangeTest(Value *V, const APInt &Lo, const APInt &Hi, bool isSigned, bool Inside)
Emit a computation of: (V >= Lo && V < Hi) if Inside is true, otherwise (V < Lo || V >= Hi).
Instruction * foldICmpBinOp(ICmpInst &Cmp, const SimplifyQuery &SQ)
Try to fold icmp (binop), X or icmp X, (binop).
Instruction * foldICmpSubConstant(ICmpInst &Cmp, BinaryOperator *Sub, const APInt &C)
Fold icmp (sub X, Y), C.
Instruction * foldICmpInstWithConstantNotInt(ICmpInst &Cmp)
Handle icmp with constant (but not simple integer constant) RHS.
Instruction * foldICmpShlConstConst(ICmpInst &I, Value *ShAmt, const APInt &C1, const APInt &C2)
Handle "(icmp eq/ne (shl AP2, A), AP1)" -> (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)).
Value * reassociateShiftAmtsOfTwoSameDirectionShifts(BinaryOperator *Sh0, const SimplifyQuery &SQ, bool AnalyzeForSignBitExtraction=false)
Instruction * foldICmpEqIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II, const APInt &C)
Fold an equality icmp with LLVM intrinsic and constant operand.
Value * foldMultiplicationOverflowCheck(ICmpInst &Cmp)
Fold (-1 u/ x) u< y ((x * y) ?/ x) != y to @llvm.
Instruction * foldICmpWithConstant(ICmpInst &Cmp)
Fold icmp Pred X, C.
CmpInst * canonicalizeICmpPredicate(CmpInst &I)
If we have a comparison with a non-canonical predicate, if we can update all the users,...
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Instruction * foldICmpWithZero(ICmpInst &Cmp)
Instruction * foldICmpCommutative(CmpPredicate Pred, Value *Op0, Value *Op1, ICmpInst &CxtI)
Instruction * foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp, BinaryOperator *BO, const APInt &C)
Fold an icmp equality instruction with binary operator LHS and constant RHS: icmp eq/ne BO,...
Instruction * foldICmpUsingBoolRange(ICmpInst &I)
If one operand of an icmp is effectively a bool (value range of {0,1}), then try to reduce patterns b...
Instruction * foldICmpWithTrunc(ICmpInst &Cmp)
Instruction * foldICmpIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II, const APInt &C)
Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
bool SimplifyDemandedBits(Instruction *I, unsigned Op, const APInt &DemandedMask, KnownBits &Known, unsigned Depth, const SimplifyQuery &Q) override
This form of SimplifyDemandedBits simplifies the specified instruction operand if possible,...
bool matchThreeWayIntCompare(SelectInst *SI, Value *&LHS, Value *&RHS, ConstantInt *&Less, ConstantInt *&Equal, ConstantInt *&Greater)
Match a select chain which produces one of three values based on whether the LHS is less than,...
Instruction * foldCmpLoadFromIndexedGlobal(LoadInst *LI, GetElementPtrInst *GEP, GlobalVariable *GV, CmpInst &ICI, ConstantInt *AndCst=nullptr)
This is called when we see this pattern: cmp pred (load (gep GV, ...)), cmpcst where GV is a global v...
Instruction * visitFCmpInst(FCmpInst &I)
Instruction * foldICmpUsingKnownBits(ICmpInst &Cmp)
Try to fold the comparison based on range information we can get by checking whether bits are known t...
Instruction * foldICmpDivConstant(ICmpInst &Cmp, BinaryOperator *Div, const APInt &C)
Fold icmp ({su}div X, Y), C.
Instruction * foldIRemByPowerOfTwoToBitTest(ICmpInst &I)
If we have: icmp eq/ne (urem/srem x, y), 0 iff y is a power-of-two, we can replace this with a bit te...
Instruction * foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, Constant *RHSC)
Fold fcmp ([us]itofp x, cst) if possible.
Instruction * foldICmpUDivConstant(ICmpInst &Cmp, BinaryOperator *UDiv, const APInt &C)
Fold icmp (udiv X, Y), C.
Instruction * foldICmpAddOpConst(Value *X, const APInt &C, CmpPredicate Pred)
Fold "icmp pred (X+C), X".
Constant * getLosslessTrunc(Constant *C, Type *TruncTy, unsigned ExtOp)
Instruction * foldICmpWithCastOp(ICmpInst &ICmp)
Handle icmp (cast x), (cast or constant).
Instruction * foldICmpTruncConstant(ICmpInst &Cmp, TruncInst *Trunc, const APInt &C)
Fold icmp (trunc X), C.
Instruction * foldICmpAddConstant(ICmpInst &Cmp, BinaryOperator *Add, const APInt &C)
Fold icmp (add X, Y), C.
Instruction * foldICmpMulConstant(ICmpInst &Cmp, BinaryOperator *Mul, const APInt &C)
Fold icmp (mul X, Y), C.
Instruction * tryFoldInstWithCtpopWithNot(Instruction *I)
Instruction * foldICmpXorConstant(ICmpInst &Cmp, BinaryOperator *Xor, const APInt &C)
Fold icmp (xor X, Y), C.
Instruction * foldSelectICmp(CmpPredicate Pred, SelectInst *SI, Value *RHS, const ICmpInst &I)
Instruction * foldICmpInstWithConstantAllowPoison(ICmpInst &Cmp, const APInt &C)
Try to fold integer comparisons with a constant operand: icmp Pred X, C where X is some kind of instr...
Instruction * foldICmpAndShift(ICmpInst &Cmp, BinaryOperator *And, const APInt &C1, const APInt &C2)
Fold icmp (and (sh X, Y), C2), C1.
Instruction * foldICmpInstWithConstant(ICmpInst &Cmp)
Try to fold integer comparisons with a constant operand: icmp Pred X, C where X is some kind of instr...
Instruction * foldICmpXorShiftConst(ICmpInst &Cmp, BinaryOperator *Xor, const APInt &C)
For power-of-2 C: ((X s>> ShiftC) ^ X) u< C --> (X + C) u< (C << 1) ((X s>> ShiftC) ^ X) u> (C - 1) -...
Instruction * foldICmpShlConstant(ICmpInst &Cmp, BinaryOperator *Shl, const APInt &C)
Fold icmp (shl X, Y), C.
Instruction * foldICmpAndConstant(ICmpInst &Cmp, BinaryOperator *And, const APInt &C)
Fold icmp (and X, Y), C.
Instruction * foldICmpEquality(ICmpInst &Cmp)
Instruction * foldICmpWithMinMax(Instruction &I, MinMaxIntrinsic *MinMax, Value *Z, CmpPredicate Pred)
Fold icmp Pred min|max(X, Y), Z.
bool dominatesAllUses(const Instruction *DI, const Instruction *UI, const BasicBlock *DB) const
True when DB dominates all uses of DI except UI.
bool foldAllocaCmp(AllocaInst *Alloca)
Instruction * visitICmpInst(ICmpInst &I)
OverflowResult computeOverflow(Instruction::BinaryOps BinaryOp, bool IsSigned, Value *LHS, Value *RHS, Instruction *CxtI) const
Instruction * foldICmpWithDominatingICmp(ICmpInst &Cmp)
Canonicalize icmp instructions based on dominating conditions.
bool replacedSelectWithOperand(SelectInst *SI, const ICmpInst *Icmp, const unsigned SIOpd)
Try to replace select with select operand SIOpd in SI-ICmp sequence.
Instruction * foldICmpShrConstConst(ICmpInst &I, Value *ShAmt, const APInt &C1, const APInt &C2)
Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" -> (icmp eq/ne A, Log2(AP2/AP1)) -> (icmp eq/ne A,...
void freelyInvertAllUsersOf(Value *V, Value *IgnoredUser=nullptr)
Freely adapt every user of V as-if V was changed to !V.
Instruction * foldICmpAndConstConst(ICmpInst &Cmp, BinaryOperator *And, const APInt &C1)
Fold icmp (and X, C2), C1.
Instruction * foldICmpBitCast(ICmpInst &Cmp)
Instruction * foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, CmpPredicate Cond, Instruction &I)
Fold comparisons between a GEP instruction and something else.
The core instruction combiner logic.
Definition: InstCombiner.h:48
OverflowResult computeOverflowForSignedSub(const Value *LHS, const Value *RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:508
SimplifyQuery SQ
Definition: InstCombiner.h:77
bool isFreeToInvert(Value *V, bool WillInvertAllUses, bool &DoesConsume)
Return true if the specified value is free to invert (apply ~ to).
Definition: InstCombiner.h:234
OverflowResult computeOverflowForUnsignedMul(const Value *LHS, const Value *RHS, const Instruction *CxtI, bool IsNSW=false) const
Definition: InstCombiner.h:471
static unsigned getComplexity(Value *V)
Assign a complexity or rank value to LLVM Values.
Definition: InstCombiner.h:143
TargetLibraryInfo & TLI
Definition: InstCombiner.h:74
bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero=false, unsigned Depth=0, const Instruction *CxtI=nullptr)
Definition: InstCombiner.h:449
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
Definition: InstCombiner.h:394
uint64_t MaxArraySizeForCombine
Maximum size of array considered when transforming.
Definition: InstCombiner.h:56
OverflowResult computeOverflowForSignedAdd(const WithCache< const Value * > &LHS, const WithCache< const Value * > &RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:494
static Constant * SubOne(Constant *C)
Subtract one from a Constant.
Definition: InstCombiner.h:183
static std::optional< std::pair< CmpPredicate, Constant * > > getFlippedStrictnessPredicateAndConstant(CmpPredicate Pred, Constant *C)
OverflowResult computeOverflowForUnsignedSub(const Value *LHS, const Value *RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:501
static bool isCanonicalPredicate(CmpPredicate Pred)
Predicate canonicalization reduces the number of patterns that need to be matched by other transforms...
Definition: InstCombiner.h:160
const DataLayout & DL
Definition: InstCombiner.h:76
DomConditionCache DC
Definition: InstCombiner.h:82
bool canFreelyInvertAllUsersOf(Instruction *V, Value *IgnoredUser)
Given i1 V, can every user of V be freely adapted if V is changed to !V ? InstCombine's freelyInvertA...
Definition: InstCombiner.h:250
AssumptionCache & AC
Definition: InstCombiner.h:73
void addToWorklist(Instruction *I)
Definition: InstCombiner.h:338
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
Definition: InstCombiner.h:418
DominatorTree & DT
Definition: InstCombiner.h:75
OverflowResult computeOverflowForSignedMul(const Value *LHS, const Value *RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:479
void computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth, const Instruction *CxtI) const
Definition: InstCombiner.h:439
BuilderTy & Builder
Definition: InstCombiner.h:61
OverflowResult computeOverflowForUnsignedAdd(const WithCache< const Value * > &LHS, const WithCache< const Value * > &RHS, const Instruction *CxtI) const
Definition: InstCombiner.h:486
Value * getFreelyInverted(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume)
Definition: InstCombiner.h:215
const SimplifyQuery & getSimplifyQuery() const
Definition: InstCombiner.h:344
unsigned ComputeMaxSignificantBits(const Value *Op, unsigned Depth=0, const Instruction *CxtI=nullptr) const
Definition: InstCombiner.h:466
bool hasNoNaNs() const LLVM_READONLY
Determine whether the no-NaNs flag is set.
bool hasNoUnsignedWrap() const LLVM_READONLY
Determine whether the no unsigned wrap flag is set.
bool hasNoInfs() const LLVM_READONLY
Determine whether the no-infs flag is set.
bool isArithmeticShift() const
Return true if this is an arithmetic shift right.
Definition: Instruction.h:323
bool hasNoSignedWrap() const LLVM_READONLY
Determine whether the no signed wrap flag is set.
bool isCommutative() const LLVM_READONLY
Return true if the instruction is commutative:
bool isExact() const LLVM_READONLY
Determine whether the exact flag is set.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:274
bool isShift() const
Definition: Instruction.h:282
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:311
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:48
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:55
An instruction for reading from memory.
Definition: Instructions.h:176
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:205
This class represents min/max intrinsics.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Represents a saturating add/sub intrinsic.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, Instruction *MDFrom=nullptr)
A vector that has set insertion semantics.
Definition: SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
Definition: SetVector.h:254
This instruction constructs a fixed permutation of two input vectors.
bool empty() const
Definition: SmallVector.h:81
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:937
void push_back(const T &Elt)
Definition: SmallVector.h:413
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
Class to represent struct types.
Definition: DerivedTypes.h:218
This class represents a truncation of integer types.
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property.
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
unsigned getIntegerBitWidth() const
const fltSemantics & getFltSemantics() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:270
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition: Type.h:243
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:264
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
Definition: Type.h:165
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
int getFPMantissaWidth() const
Return the width of the mantissa of this type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:237
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:355
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
void setOperand(unsigned i, Value *Val)
Definition: User.h:233
Value * getOperand(unsigned i) const
Definition: User.h:228
unsigned getNumOperands() const
Definition: User.h:250
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr) const
Accumulate the constant offset this value has compared to a base pointer.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
iterator_range< user_iterator > users()
Definition: Value.h:421
bool hasNUsesOrMore(unsigned N) const
Return true if this value has N uses or more.
Definition: Value.cpp:153
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:694
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
iterator_range< use_iterator > uses()
Definition: Value.h:376
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:383
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:202
const ParentTy * getParent() const
Definition: ilist_node.h:32
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt RoundingUDiv(const APInt &A, const APInt &B, APInt::Rounding RM)
Return A unsign-divided by B, rounded by the given rounding mode.
Definition: APInt.cpp:2736
APInt RoundingSDiv(const APInt &A, const APInt &B, APInt::Rounding RM)
Return A sign-divided by B, rounded by the given rounding mode.
Definition: APInt.cpp:2754
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition: Intrinsics.cpp:731
cst_pred_ty< is_all_ones > m_AllOnes()
Match an integer or vector with all bits set.
Definition: PatternMatch.h:524
cst_pred_ty< is_lowbit_mask > m_LowBitMask()
Match an integer or vector with only the low bit(s) set.
Definition: PatternMatch.h:673
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
Definition: PatternMatch.h:100
cst_pred_ty< is_sign_mask > m_SignMask()
Match an integer or vector with only the sign bit(s) set.
Definition: PatternMatch.h:664
OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
cst_pred_ty< is_power2 > m_Power2()
Match an integer or vector power-of-2.
Definition: PatternMatch.h:619
BinaryOp_match< LHS, RHS, Instruction::URem > m_URem(const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, TruncInst >, OpTy > m_TruncOrSelf(const OpTy &Op)
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
Definition: PatternMatch.h:165
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
Definition: PatternMatch.h:982
match_combine_or< CastInst_match< OpTy, ZExtInst >, OpTy > m_ZExtOrSelf(const OpTy &Op)
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
BinOpPred_match< LHS, RHS, is_idiv_op > m_IDiv(const LHS &L, const RHS &R)
Matches integer division operations.
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
Definition: PatternMatch.h:826
cstfp_pred_ty< is_any_zero_fp > m_AnyZeroFP()
Match a floating-point negative zero or positive zero.
Definition: PatternMatch.h:764
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:885
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
specific_intval< true > m_SpecificIntAllowPoison(const APInt &V)
Definition: PatternMatch.h:990
CmpClass_match< LHS, RHS, ICmpInst, true > m_c_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
Matches an ICmp with a predicate over LHS and RHS in either order.
OverflowingBinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub, OverflowingBinaryOperator::NoSignedWrap > m_NSWNeg(const ValTy &V)
Matches a 'Neg' as 'sub nsw 0, V'.
cst_pred_ty< is_nonnegative > m_NonNegative()
Match an integer or vector of non-negative values.
Definition: PatternMatch.h:560
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
Definition: PatternMatch.h:168
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
Definition: PatternMatch.h:592
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
BinOpPred_match< LHS, RHS, is_logical_shift_op > m_LogicalShift(const LHS &L, const RHS &R)
Matches logical shift operations.
match_combine_and< LTy, RTy > m_CombineAnd(const LTy &L, const RTy &R)
Combine two pattern matchers matching L && R.
Definition: PatternMatch.h:245
MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > m_SMin(const LHS &L, const RHS &R)
m_Intrinsic_Ty< Opnd0 >::Ty m_Sqrt(const Opnd0 &Op0)
BinaryOp_match< LHS, RHS, Instruction::Xor, true > m_c_Xor(const LHS &L, const RHS &R)
Matches an Xor with LHS and RHS in either order.
BinaryOp_match< LHS, RHS, Instruction::FAdd > m_FAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
Definition: PatternMatch.h:903
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
Definition: PatternMatch.h:599
apint_match m_APIntAllowPoison(const APInt *&Res)
Match APInt while allowing poison in splat vector constants.
Definition: PatternMatch.h:305
NoWrapTrunc_match< OpTy, TruncInst::NoSignedWrap > m_NSWTrunc(const OpTy &Op)
Matches trunc nsw.
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
match_combine_and< class_match< Constant >, match_unless< constantexpr_match > > m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
Definition: PatternMatch.h:864
ThreeOps_match< decltype(m_Value()), LHS, RHS, Instruction::Select, true > m_c_Select(const LHS &L, const RHS &R)
Match Select(C, LHS, RHS) or Select(C, RHS, LHS)
CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Shl, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWShl(const LHS &L, const RHS &R)
OverflowingBinaryOp_match< LHS, RHS, Instruction::Mul, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWMul(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::UDiv > m_UDiv(const LHS &L, const RHS &R)
cst_pred_ty< is_negated_power2_or_zero > m_NegatedPower2OrZero()
Match a integer or vector negated power-of-2.
Definition: PatternMatch.h:639
NoWrapTrunc_match< OpTy, TruncInst::NoUnsignedWrap > m_NUWTrunc(const OpTy &Op)
Matches trunc nuw.
cst_pred_ty< custom_checkfn< APInt > > m_CheckedInt(function_ref< bool(const APInt &)> CheckFn)
Match an integer or vector where CheckFn(ele) for each element is true.
Definition: PatternMatch.h:481
cst_pred_ty< is_lowbit_mask_or_zero > m_LowBitMaskOrZero()
Match an integer or vector with only the low bit(s) set.
Definition: PatternMatch.h:683
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
match_combine_or< BinaryOp_match< LHS, RHS, Instruction::Add >, DisjointOr_match< LHS, RHS > > m_AddLike(const LHS &L, const RHS &R)
Match either "add" or "or disjoint".
CastInst_match< OpTy, UIToFPInst > m_UIToFP(const OpTy &Op)
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
BinaryOp_match< LHS, RHS, Instruction::SDiv > m_SDiv(const LHS &L, const RHS &R)
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
Definition: PatternMatch.h:299
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
Signum_match< Val_t > m_Signum(const Val_t &V)
Matches a signum pattern.
CastInst_match< OpTy, SIToFPInst > m_SIToFP(const OpTy &Op)
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
cstfp_pred_ty< is_pos_zero_fp > m_PosZeroFP()
Match a floating-point positive zero.
Definition: PatternMatch.h:773
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
UAddWithOverflow_match< LHS_t, RHS_t, Sum_t > m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S)
Match an icmp instruction checking for unsigned overflow on addition.
m_Intrinsic_Ty< Opnd0 >::Ty m_VecReverse(const Opnd0 &Op0)
BinOpPred_match< LHS, RHS, is_irem_op > m_IRem(const LHS &L, const RHS &R)
Matches integer remainder operations.
apfloat_match m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
Definition: PatternMatch.h:316
match_combine_or< match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > >, match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > > > m_MaxOrMin(const LHS &L, const RHS &R)
CastInst_match< OpTy, FPTruncInst > m_FPTrunc(const OpTy &Op)
auto m_Undef()
Match an arbitrary undef constant.
Definition: PatternMatch.h:152
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
Definition: PatternMatch.h:612
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
ElementWiseBitCast_match< OpTy > m_ElementWiseBitCast(const OpTy &Op)
m_Intrinsic_Ty< Opnd0 >::Ty m_FAbs(const Opnd0 &Op0)
BinaryOp_match< LHS, RHS, Instruction::Mul, true > m_c_Mul(const LHS &L, const RHS &R)
Matches a Mul with LHS and RHS in either order.
CastOperator_match< OpTy, Instruction::PtrToInt > m_PtrToInt(const OpTy &Op)
Matches PtrToInt.
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
match_unless< Ty > m_Unless(const Ty &M)
Match if the inner matcher does NOT match.
Definition: PatternMatch.h:203
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
Definition: PatternMatch.h:239
cst_pred_ty< icmp_pred_with_threshold > m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold)
Match an integer or vector with every element comparing 'pred' (eg/ne/...) to Threshold.
Definition: PatternMatch.h:698
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:480
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition: STLExtras.h:854
OverflowResult
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1739
bool isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS, bool &TrueIfSigned)
Given an exploded icmp instruction, return true if the comparison only checks the sign bit.
bool isKnownNeverInfinity(const Value *V, unsigned Depth, const SimplifyQuery &SQ)
Return true if the floating-point scalar value is not an infinity or if the floating-point vector val...
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:657
bool isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL, bool OrZero=false, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given value is known to have exactly one bit set when defined.
Value * simplifyFCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q)
Given operands for an FCmpInst, fold the result or return null.
ConstantRange computeConstantRange(const Value *V, bool ForSigned, bool UseInstrInfo=true, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Determine the possible constant range of an integer or vector of integer value.
Constant * ConstantFoldExtractValueInstruction(Constant *Agg, ArrayRef< unsigned > Idxs)
Attempt to constant fold an extractvalue instruction with the specified operands and indices.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
Value * simplifyAddInst(Value *LHS, Value *RHS, bool IsNSW, bool IsNUW, const SimplifyQuery &Q)
Given operands for an Add, fold the result or return null.
Constant * ConstantFoldConstant(const Constant *C, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldConstant - Fold the constant using the specified DataLayout.
bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:340
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition: bit.h:281
Value * emitGEPOffset(IRBuilderBase *Builder, const DataLayout &DL, User *GEP, bool NoAssumptions=false)
Given a getelementptr instruction/constantexpr, emit the code necessary to compute the offset from th...
Definition: Local.cpp:22
constexpr unsigned MaxAnalysisRecursionDepth
Definition: ValueTracking.h:44
Constant * ConstantFoldUnaryOpOperand(unsigned Opcode, Constant *Op, const DataLayout &DL)
Attempt to constant fold a unary operation with the specified operand.
bool isKnownNegative(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the given value is known be negative (i.e.
SelectPatternFlavor
Specific patterns of select instructions we can match.
@ SPF_UNKNOWN
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
bool PointerMayBeCaptured(const Value *V, bool ReturnCaptures, bool StoreCaptures, unsigned MaxUsesToExplore=0)
PointerMayBeCaptured - Return true if this pointer value may be captured by the enclosing function (w...
SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
Definition: Function.cpp:1187
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1753
Value * simplifyICmpInst(CmpPredicate Pred, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for an ICmpInst, fold the result or return null.
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
Constant * ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL)
Attempt to constant fold a binary operation with the specified operands.
bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
@ Other
Any other memory.
bool isKnownNonEqual(const Value *V1, const Value *V2, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given values are known to be non-equal when defined.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
@ Add
Sum of integers.
@ UMax
Unsigned integer max implemented in terms of select(cmp()).
std::optional< DecomposedBitTest > decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate Pred, bool LookThroughTrunc=true, bool AllowNonZeroC=false)
Decompose an icmp into the form ((X & Mask) pred C) if possible.
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
DWARFExpression::Operation Op
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:217
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition: STLExtras.h:1945
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition: STLExtras.h:2087
bool isKnownNeverNaN(const Value *V, unsigned Depth, const SimplifyQuery &SQ)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
bool isKnownPositive(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the given value is known be positive (i.e.
bool isKnownNonNegative(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the give value is known to be non-negative.
std::optional< bool > isImpliedCondition(const Value *LHS, const Value *RHS, const DataLayout &DL, bool LHSIsTrue=true, unsigned Depth=0)
Return true if RHS is known to be implied true by LHS.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define NC
Definition: regutils.h:42
static constexpr roundingMode rmNearestTiesToEven
Definition: APFloat.h:297
static constexpr roundingMode rmTowardZero
Definition: APFloat.h:301
This callback is used in conjunction with PointerMayBeCaptured.
Represent subnormal handling kind for floating point instruction inputs and outputs.
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ PositiveZero
Denormals are flushed to positive zero.
static constexpr DenormalMode getIEEE()
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition: KnownBits.h:100
bool isZero() const
Returns true if value is all zero.
Definition: KnownBits.h:79
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition: KnownBits.h:234
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
Definition: KnownBits.h:266
APInt getSignedMaxValue() const
Return the maximal signed value possible given these KnownBits.
Definition: KnownBits.h:143
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition: KnownBits.h:281
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:43
bool isConstant() const
Returns true if we know the value of all bits.
Definition: KnownBits.h:53
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:240
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition: KnownBits.h:137
APInt getMinValue() const
Return the minimal unsigned value possible given these KnownBits.
Definition: KnownBits.h:121
bool isStrictlyPositive() const
Returns true if this value is known to be positive.
Definition: KnownBits.h:106
bool isNegative() const
Returns true if this value is known to be negative.
Definition: KnownBits.h:97
unsigned countMinPopulation() const
Returns the number of bits known to be one.
Definition: KnownBits.h:278
APInt getSignedMinValue() const
Return the minimal signed value possible given these KnownBits.
Definition: KnownBits.h:127
const APInt & getConstant() const
Returns the value when all bits have a known value.
Definition: KnownBits.h:59
SelectPatternFlavor Flavor
static bool isMinOrMax(SelectPatternFlavor SPF)
When implementing this min/max pattern as fcmp; select, does the fcmp have to be ordered?
const DataLayout & DL
Definition: SimplifyQuery.h:71
const Instruction * CxtI
Definition: SimplifyQuery.h:75
const DominatorTree * DT
Definition: SimplifyQuery.h:73
SimplifyQuery getWithInstruction(const Instruction *I) const
AssumptionCache * AC
Definition: SimplifyQuery.h:74
SimplifyQuery getWithoutDomCondCache() const
A MapVector that performs no allocations if smaller than a certain size.
Definition: MapVector.h:254