LLVM 20.0.0git
InstructionCombining.cpp
Go to the documentation of this file.
1//===- InstructionCombining.cpp - Combine multiple instructions -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// InstructionCombining - Combine instructions to form fewer, simple
10// instructions. This pass does not modify the CFG. This pass is where
11// algebraic simplification happens.
12//
13// This pass combines things like:
14// %Y = add i32 %X, 1
15// %Z = add i32 %Y, 1
16// into:
17// %Z = add i32 %X, 2
18//
19// This is a simple worklist driven algorithm.
20//
21// This pass guarantees that the following canonicalizations are performed on
22// the program:
23// 1. If a binary operator has a constant operand, it is moved to the RHS
24// 2. Bitwise operators with constant operands are always grouped so that
25// shifts are performed first, then or's, then and's, then xor's.
26// 3. Compare instructions are converted from <,>,<=,>= to ==,!= if possible
27// 4. All cmp instructions on boolean values are replaced with logical ops
28// 5. add X, X is represented as (X*2) => (X << 1)
29// 6. Multiplies with a power-of-two constant argument are transformed into
30// shifts.
31// ... etc.
32//
33//===----------------------------------------------------------------------===//
34
35#include "InstCombineInternal.h"
36#include "llvm/ADT/APInt.h"
37#include "llvm/ADT/ArrayRef.h"
38#include "llvm/ADT/DenseMap.h"
41#include "llvm/ADT/Statistic.h"
46#include "llvm/Analysis/CFG.h"
61#include "llvm/IR/BasicBlock.h"
62#include "llvm/IR/CFG.h"
63#include "llvm/IR/Constant.h"
64#include "llvm/IR/Constants.h"
65#include "llvm/IR/DIBuilder.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
69#include "llvm/IR/Dominators.h"
71#include "llvm/IR/Function.h"
73#include "llvm/IR/IRBuilder.h"
74#include "llvm/IR/InstrTypes.h"
75#include "llvm/IR/Instruction.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/Metadata.h"
80#include "llvm/IR/Operator.h"
81#include "llvm/IR/PassManager.h"
83#include "llvm/IR/Type.h"
84#include "llvm/IR/Use.h"
85#include "llvm/IR/User.h"
86#include "llvm/IR/Value.h"
87#include "llvm/IR/ValueHandle.h"
92#include "llvm/Support/Debug.h"
100#include <algorithm>
101#include <cassert>
102#include <cstdint>
103#include <memory>
104#include <optional>
105#include <string>
106#include <utility>
107
108#define DEBUG_TYPE "instcombine"
110#include <optional>
111
112using namespace llvm;
113using namespace llvm::PatternMatch;
114
115STATISTIC(NumWorklistIterations,
116 "Number of instruction combining iterations performed");
117STATISTIC(NumOneIteration, "Number of functions with one iteration");
118STATISTIC(NumTwoIterations, "Number of functions with two iterations");
119STATISTIC(NumThreeIterations, "Number of functions with three iterations");
120STATISTIC(NumFourOrMoreIterations,
121 "Number of functions with four or more iterations");
122
123STATISTIC(NumCombined , "Number of insts combined");
124STATISTIC(NumConstProp, "Number of constant folds");
125STATISTIC(NumDeadInst , "Number of dead inst eliminated");
126STATISTIC(NumSunkInst , "Number of instructions sunk");
127STATISTIC(NumExpand, "Number of expansions");
128STATISTIC(NumFactor , "Number of factorizations");
129STATISTIC(NumReassoc , "Number of reassociations");
130DEBUG_COUNTER(VisitCounter, "instcombine-visit",
131 "Controls which instructions are visited");
132
133static cl::opt<bool>
134EnableCodeSinking("instcombine-code-sinking", cl::desc("Enable code sinking"),
135 cl::init(true));
136
138 "instcombine-max-sink-users", cl::init(32),
139 cl::desc("Maximum number of undroppable users for instruction sinking"));
140
142MaxArraySize("instcombine-maxarray-size", cl::init(1024),
143 cl::desc("Maximum array size considered when doing a combine"));
144
145// FIXME: Remove this flag when it is no longer necessary to convert
146// llvm.dbg.declare to avoid inaccurate debug info. Setting this to false
147// increases variable availability at the cost of accuracy. Variables that
148// cannot be promoted by mem2reg or SROA will be described as living in memory
149// for their entire lifetime. However, passes like DSE and instcombine can
150// delete stores to the alloca, leading to misleading and inaccurate debug
151// information. This flag can be removed when those passes are fixed.
152static cl::opt<unsigned> ShouldLowerDbgDeclare("instcombine-lower-dbg-declare",
153 cl::Hidden, cl::init(true));
154
155std::optional<Instruction *>
157 // Handle target specific intrinsics
158 if (II.getCalledFunction()->isTargetIntrinsic()) {
159 return TTIForTargetIntrinsicsOnly.instCombineIntrinsic(*this, II);
160 }
161 return std::nullopt;
162}
163
165 IntrinsicInst &II, APInt DemandedMask, KnownBits &Known,
166 bool &KnownBitsComputed) {
167 // Handle target specific intrinsics
168 if (II.getCalledFunction()->isTargetIntrinsic()) {
169 return TTIForTargetIntrinsicsOnly.simplifyDemandedUseBitsIntrinsic(
170 *this, II, DemandedMask, Known, KnownBitsComputed);
171 }
172 return std::nullopt;
173}
174
176 IntrinsicInst &II, APInt DemandedElts, APInt &PoisonElts,
177 APInt &PoisonElts2, APInt &PoisonElts3,
178 std::function<void(Instruction *, unsigned, APInt, APInt &)>
179 SimplifyAndSetOp) {
180 // Handle target specific intrinsics
181 if (II.getCalledFunction()->isTargetIntrinsic()) {
182 return TTIForTargetIntrinsicsOnly.simplifyDemandedVectorEltsIntrinsic(
183 *this, II, DemandedElts, PoisonElts, PoisonElts2, PoisonElts3,
184 SimplifyAndSetOp);
185 }
186 return std::nullopt;
187}
188
189bool InstCombiner::isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const {
190 // Approved exception for TTI use: This queries a legality property of the
191 // target, not an profitability heuristic. Ideally this should be part of
192 // DataLayout instead.
193 return TTIForTargetIntrinsicsOnly.isValidAddrSpaceCast(FromAS, ToAS);
194}
195
196Value *InstCombinerImpl::EmitGEPOffset(GEPOperator *GEP, bool RewriteGEP) {
197 if (!RewriteGEP)
199
201 auto *Inst = dyn_cast<Instruction>(GEP);
202 if (Inst)
204
205 Value *Offset = EmitGEPOffset(GEP);
206 // If a non-trivial GEP has other uses, rewrite it to avoid duplicating
207 // the offset arithmetic.
208 if (Inst && !GEP->hasOneUse() && !GEP->hasAllConstantIndices() &&
209 !GEP->getSourceElementType()->isIntegerTy(8)) {
211 *Inst, Builder.CreateGEP(Builder.getInt8Ty(), GEP->getPointerOperand(),
212 Offset, "", GEP->getNoWrapFlags()));
214 }
215 return Offset;
216}
217
218/// Legal integers and common types are considered desirable. This is used to
219/// avoid creating instructions with types that may not be supported well by the
220/// the backend.
221/// NOTE: This treats i8, i16 and i32 specially because they are common
222/// types in frontend languages.
223bool InstCombinerImpl::isDesirableIntType(unsigned BitWidth) const {
224 switch (BitWidth) {
225 case 8:
226 case 16:
227 case 32:
228 return true;
229 default:
230 return DL.isLegalInteger(BitWidth);
231 }
232}
233
234/// Return true if it is desirable to convert an integer computation from a
235/// given bit width to a new bit width.
236/// We don't want to convert from a legal or desirable type (like i8) to an
237/// illegal type or from a smaller to a larger illegal type. A width of '1'
238/// is always treated as a desirable type because i1 is a fundamental type in
239/// IR, and there are many specialized optimizations for i1 types.
240/// Common/desirable widths are equally treated as legal to convert to, in
241/// order to open up more combining opportunities.
242bool InstCombinerImpl::shouldChangeType(unsigned FromWidth,
243 unsigned ToWidth) const {
244 bool FromLegal = FromWidth == 1 || DL.isLegalInteger(FromWidth);
245 bool ToLegal = ToWidth == 1 || DL.isLegalInteger(ToWidth);
246
247 // Convert to desirable widths even if they are not legal types.
248 // Only shrink types, to prevent infinite loops.
249 if (ToWidth < FromWidth && isDesirableIntType(ToWidth))
250 return true;
251
252 // If this is a legal or desiable integer from type, and the result would be
253 // an illegal type, don't do the transformation.
254 if ((FromLegal || isDesirableIntType(FromWidth)) && !ToLegal)
255 return false;
256
257 // Otherwise, if both are illegal, do not increase the size of the result. We
258 // do allow things like i160 -> i64, but not i64 -> i160.
259 if (!FromLegal && !ToLegal && ToWidth > FromWidth)
260 return false;
261
262 return true;
263}
264
265/// Return true if it is desirable to convert a computation from 'From' to 'To'.
266/// We don't want to convert from a legal to an illegal type or from a smaller
267/// to a larger illegal type. i1 is always treated as a legal type because it is
268/// a fundamental type in IR, and there are many specialized optimizations for
269/// i1 types.
270bool InstCombinerImpl::shouldChangeType(Type *From, Type *To) const {
271 // TODO: This could be extended to allow vectors. Datalayout changes might be
272 // needed to properly support that.
273 if (!From->isIntegerTy() || !To->isIntegerTy())
274 return false;
275
276 unsigned FromWidth = From->getPrimitiveSizeInBits();
277 unsigned ToWidth = To->getPrimitiveSizeInBits();
278 return shouldChangeType(FromWidth, ToWidth);
279}
280
281// Return true, if No Signed Wrap should be maintained for I.
282// The No Signed Wrap flag can be kept if the operation "B (I.getOpcode) C",
283// where both B and C should be ConstantInts, results in a constant that does
284// not overflow. This function only handles the Add and Sub opcodes. For
285// all other opcodes, the function conservatively returns false.
287 auto *OBO = dyn_cast<OverflowingBinaryOperator>(&I);
288 if (!OBO || !OBO->hasNoSignedWrap())
289 return false;
290
291 // We reason about Add and Sub Only.
292 Instruction::BinaryOps Opcode = I.getOpcode();
293 if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
294 return false;
295
296 const APInt *BVal, *CVal;
297 if (!match(B, m_APInt(BVal)) || !match(C, m_APInt(CVal)))
298 return false;
299
300 bool Overflow = false;
301 if (Opcode == Instruction::Add)
302 (void)BVal->sadd_ov(*CVal, Overflow);
303 else
304 (void)BVal->ssub_ov(*CVal, Overflow);
305
306 return !Overflow;
307}
308
310 auto *OBO = dyn_cast<OverflowingBinaryOperator>(&I);
311 return OBO && OBO->hasNoUnsignedWrap();
312}
313
315 auto *OBO = dyn_cast<OverflowingBinaryOperator>(&I);
316 return OBO && OBO->hasNoSignedWrap();
317}
318
319/// Conservatively clears subclassOptionalData after a reassociation or
320/// commutation. We preserve fast-math flags when applicable as they can be
321/// preserved.
323 FPMathOperator *FPMO = dyn_cast<FPMathOperator>(&I);
324 if (!FPMO) {
325 I.clearSubclassOptionalData();
326 return;
327 }
328
329 FastMathFlags FMF = I.getFastMathFlags();
330 I.clearSubclassOptionalData();
331 I.setFastMathFlags(FMF);
332}
333
334/// Combine constant operands of associative operations either before or after a
335/// cast to eliminate one of the associative operations:
336/// (op (cast (op X, C2)), C1) --> (cast (op X, op (C1, C2)))
337/// (op (cast (op X, C2)), C1) --> (op (cast X), op (C1, C2))
339 InstCombinerImpl &IC) {
340 auto *Cast = dyn_cast<CastInst>(BinOp1->getOperand(0));
341 if (!Cast || !Cast->hasOneUse())
342 return false;
343
344 // TODO: Enhance logic for other casts and remove this check.
345 auto CastOpcode = Cast->getOpcode();
346 if (CastOpcode != Instruction::ZExt)
347 return false;
348
349 // TODO: Enhance logic for other BinOps and remove this check.
350 if (!BinOp1->isBitwiseLogicOp())
351 return false;
352
353 auto AssocOpcode = BinOp1->getOpcode();
354 auto *BinOp2 = dyn_cast<BinaryOperator>(Cast->getOperand(0));
355 if (!BinOp2 || !BinOp2->hasOneUse() || BinOp2->getOpcode() != AssocOpcode)
356 return false;
357
358 Constant *C1, *C2;
359 if (!match(BinOp1->getOperand(1), m_Constant(C1)) ||
360 !match(BinOp2->getOperand(1), m_Constant(C2)))
361 return false;
362
363 // TODO: This assumes a zext cast.
364 // Eg, if it was a trunc, we'd cast C1 to the source type because casting C2
365 // to the destination type might lose bits.
366
367 // Fold the constants together in the destination type:
368 // (op (cast (op X, C2)), C1) --> (op (cast X), FoldedC)
369 const DataLayout &DL = IC.getDataLayout();
370 Type *DestTy = C1->getType();
371 Constant *CastC2 = ConstantFoldCastOperand(CastOpcode, C2, DestTy, DL);
372 if (!CastC2)
373 return false;
374 Constant *FoldedC = ConstantFoldBinaryOpOperands(AssocOpcode, C1, CastC2, DL);
375 if (!FoldedC)
376 return false;
377
378 IC.replaceOperand(*Cast, 0, BinOp2->getOperand(0));
379 IC.replaceOperand(*BinOp1, 1, FoldedC);
381 Cast->dropPoisonGeneratingFlags();
382 return true;
383}
384
385// Simplifies IntToPtr/PtrToInt RoundTrip Cast.
386// inttoptr ( ptrtoint (x) ) --> x
387Value *InstCombinerImpl::simplifyIntToPtrRoundTripCast(Value *Val) {
388 auto *IntToPtr = dyn_cast<IntToPtrInst>(Val);
389 if (IntToPtr && DL.getTypeSizeInBits(IntToPtr->getDestTy()) ==
390 DL.getTypeSizeInBits(IntToPtr->getSrcTy())) {
391 auto *PtrToInt = dyn_cast<PtrToIntInst>(IntToPtr->getOperand(0));
392 Type *CastTy = IntToPtr->getDestTy();
393 if (PtrToInt &&
394 CastTy->getPointerAddressSpace() ==
395 PtrToInt->getSrcTy()->getPointerAddressSpace() &&
396 DL.getTypeSizeInBits(PtrToInt->getSrcTy()) ==
397 DL.getTypeSizeInBits(PtrToInt->getDestTy()))
398 return PtrToInt->getOperand(0);
399 }
400 return nullptr;
401}
402
403/// This performs a few simplifications for operators that are associative or
404/// commutative:
405///
406/// Commutative operators:
407///
408/// 1. Order operands such that they are listed from right (least complex) to
409/// left (most complex). This puts constants before unary operators before
410/// binary operators.
411///
412/// Associative operators:
413///
414/// 2. Transform: "(A op B) op C" ==> "A op (B op C)" if "B op C" simplifies.
415/// 3. Transform: "A op (B op C)" ==> "(A op B) op C" if "A op B" simplifies.
416///
417/// Associative and commutative operators:
418///
419/// 4. Transform: "(A op B) op C" ==> "(C op A) op B" if "C op A" simplifies.
420/// 5. Transform: "A op (B op C)" ==> "B op (C op A)" if "C op A" simplifies.
421/// 6. Transform: "(A op C1) op (B op C2)" ==> "(A op B) op (C1 op C2)"
422/// if C1 and C2 are constants.
424 Instruction::BinaryOps Opcode = I.getOpcode();
425 bool Changed = false;
426
427 do {
428 // Order operands such that they are listed from right (least complex) to
429 // left (most complex). This puts constants before unary operators before
430 // binary operators.
431 if (I.isCommutative() && getComplexity(I.getOperand(0)) <
432 getComplexity(I.getOperand(1)))
433 Changed = !I.swapOperands();
434
435 if (I.isCommutative()) {
436 if (auto Pair = matchSymmetricPair(I.getOperand(0), I.getOperand(1))) {
437 replaceOperand(I, 0, Pair->first);
438 replaceOperand(I, 1, Pair->second);
439 Changed = true;
440 }
441 }
442
443 BinaryOperator *Op0 = dyn_cast<BinaryOperator>(I.getOperand(0));
444 BinaryOperator *Op1 = dyn_cast<BinaryOperator>(I.getOperand(1));
445
446 if (I.isAssociative()) {
447 // Transform: "(A op B) op C" ==> "A op (B op C)" if "B op C" simplifies.
448 if (Op0 && Op0->getOpcode() == Opcode) {
449 Value *A = Op0->getOperand(0);
450 Value *B = Op0->getOperand(1);
451 Value *C = I.getOperand(1);
452
453 // Does "B op C" simplify?
454 if (Value *V = simplifyBinOp(Opcode, B, C, SQ.getWithInstruction(&I))) {
455 // It simplifies to V. Form "A op V".
456 replaceOperand(I, 0, A);
457 replaceOperand(I, 1, V);
458 bool IsNUW = hasNoUnsignedWrap(I) && hasNoUnsignedWrap(*Op0);
459 bool IsNSW = maintainNoSignedWrap(I, B, C) && hasNoSignedWrap(*Op0);
460
461 // Conservatively clear all optional flags since they may not be
462 // preserved by the reassociation. Reset nsw/nuw based on the above
463 // analysis.
465
466 // Note: this is only valid because SimplifyBinOp doesn't look at
467 // the operands to Op0.
468 if (IsNUW)
469 I.setHasNoUnsignedWrap(true);
470
471 if (IsNSW)
472 I.setHasNoSignedWrap(true);
473
474 Changed = true;
475 ++NumReassoc;
476 continue;
477 }
478 }
479
480 // Transform: "A op (B op C)" ==> "(A op B) op C" if "A op B" simplifies.
481 if (Op1 && Op1->getOpcode() == Opcode) {
482 Value *A = I.getOperand(0);
483 Value *B = Op1->getOperand(0);
484 Value *C = Op1->getOperand(1);
485
486 // Does "A op B" simplify?
487 if (Value *V = simplifyBinOp(Opcode, A, B, SQ.getWithInstruction(&I))) {
488 // It simplifies to V. Form "V op C".
489 replaceOperand(I, 0, V);
490 replaceOperand(I, 1, C);
491 // Conservatively clear the optional flags, since they may not be
492 // preserved by the reassociation.
494 Changed = true;
495 ++NumReassoc;
496 continue;
497 }
498 }
499 }
500
501 if (I.isAssociative() && I.isCommutative()) {
502 if (simplifyAssocCastAssoc(&I, *this)) {
503 Changed = true;
504 ++NumReassoc;
505 continue;
506 }
507
508 // Transform: "(A op B) op C" ==> "(C op A) op B" if "C op A" simplifies.
509 if (Op0 && Op0->getOpcode() == Opcode) {
510 Value *A = Op0->getOperand(0);
511 Value *B = Op0->getOperand(1);
512 Value *C = I.getOperand(1);
513
514 // Does "C op A" simplify?
515 if (Value *V = simplifyBinOp(Opcode, C, A, SQ.getWithInstruction(&I))) {
516 // It simplifies to V. Form "V op B".
517 replaceOperand(I, 0, V);
518 replaceOperand(I, 1, B);
519 // Conservatively clear the optional flags, since they may not be
520 // preserved by the reassociation.
522 Changed = true;
523 ++NumReassoc;
524 continue;
525 }
526 }
527
528 // Transform: "A op (B op C)" ==> "B op (C op A)" if "C op A" simplifies.
529 if (Op1 && Op1->getOpcode() == Opcode) {
530 Value *A = I.getOperand(0);
531 Value *B = Op1->getOperand(0);
532 Value *C = Op1->getOperand(1);
533
534 // Does "C op A" simplify?
535 if (Value *V = simplifyBinOp(Opcode, C, A, SQ.getWithInstruction(&I))) {
536 // It simplifies to V. Form "B op V".
537 replaceOperand(I, 0, B);
538 replaceOperand(I, 1, V);
539 // Conservatively clear the optional flags, since they may not be
540 // preserved by the reassociation.
542 Changed = true;
543 ++NumReassoc;
544 continue;
545 }
546 }
547
548 // Transform: "(A op C1) op (B op C2)" ==> "(A op B) op (C1 op C2)"
549 // if C1 and C2 are constants.
550 Value *A, *B;
551 Constant *C1, *C2, *CRes;
552 if (Op0 && Op1 &&
553 Op0->getOpcode() == Opcode && Op1->getOpcode() == Opcode &&
554 match(Op0, m_OneUse(m_BinOp(m_Value(A), m_Constant(C1)))) &&
555 match(Op1, m_OneUse(m_BinOp(m_Value(B), m_Constant(C2)))) &&
556 (CRes = ConstantFoldBinaryOpOperands(Opcode, C1, C2, DL))) {
557 bool IsNUW = hasNoUnsignedWrap(I) &&
558 hasNoUnsignedWrap(*Op0) &&
559 hasNoUnsignedWrap(*Op1);
560 BinaryOperator *NewBO = (IsNUW && Opcode == Instruction::Add) ?
561 BinaryOperator::CreateNUW(Opcode, A, B) :
562 BinaryOperator::Create(Opcode, A, B);
563
564 if (isa<FPMathOperator>(NewBO)) {
565 FastMathFlags Flags = I.getFastMathFlags() &
566 Op0->getFastMathFlags() &
567 Op1->getFastMathFlags();
568 NewBO->setFastMathFlags(Flags);
569 }
570 InsertNewInstWith(NewBO, I.getIterator());
571 NewBO->takeName(Op1);
572 replaceOperand(I, 0, NewBO);
573 replaceOperand(I, 1, CRes);
574 // Conservatively clear the optional flags, since they may not be
575 // preserved by the reassociation.
577 if (IsNUW)
578 I.setHasNoUnsignedWrap(true);
579
580 Changed = true;
581 continue;
582 }
583 }
584
585 // No further simplifications.
586 return Changed;
587 } while (true);
588}
589
590/// Return whether "X LOp (Y ROp Z)" is always equal to
591/// "(X LOp Y) ROp (X LOp Z)".
594 // X & (Y | Z) <--> (X & Y) | (X & Z)
595 // X & (Y ^ Z) <--> (X & Y) ^ (X & Z)
596 if (LOp == Instruction::And)
597 return ROp == Instruction::Or || ROp == Instruction::Xor;
598
599 // X | (Y & Z) <--> (X | Y) & (X | Z)
600 if (LOp == Instruction::Or)
601 return ROp == Instruction::And;
602
603 // X * (Y + Z) <--> (X * Y) + (X * Z)
604 // X * (Y - Z) <--> (X * Y) - (X * Z)
605 if (LOp == Instruction::Mul)
606 return ROp == Instruction::Add || ROp == Instruction::Sub;
607
608 return false;
609}
610
611/// Return whether "(X LOp Y) ROp Z" is always equal to
612/// "(X ROp Z) LOp (Y ROp Z)".
616 return leftDistributesOverRight(ROp, LOp);
617
618 // (X {&|^} Y) >> Z <--> (X >> Z) {&|^} (Y >> Z) for all shifts.
620
621 // TODO: It would be nice to handle division, aka "(X + Y)/Z = X/Z + Y/Z",
622 // but this requires knowing that the addition does not overflow and other
623 // such subtleties.
624}
625
626/// This function returns identity value for given opcode, which can be used to
627/// factor patterns like (X * 2) + X ==> (X * 2) + (X * 1) ==> X * (2 + 1).
629 if (isa<Constant>(V))
630 return nullptr;
631
632 return ConstantExpr::getBinOpIdentity(Opcode, V->getType());
633}
634
635/// This function predicates factorization using distributive laws. By default,
636/// it just returns the 'Op' inputs. But for special-cases like
637/// 'add(shl(X, 5), ...)', this function will have TopOpcode == Instruction::Add
638/// and Op = shl(X, 5). The 'shl' is treated as the more general 'mul X, 32' to
639/// allow more factorization opportunities.
642 Value *&LHS, Value *&RHS, BinaryOperator *OtherOp) {
643 assert(Op && "Expected a binary operator");
644 LHS = Op->getOperand(0);
645 RHS = Op->getOperand(1);
646 if (TopOpcode == Instruction::Add || TopOpcode == Instruction::Sub) {
647 Constant *C;
648 if (match(Op, m_Shl(m_Value(), m_ImmConstant(C)))) {
649 // X << C --> X * (1 << C)
651 Instruction::Shl, ConstantInt::get(Op->getType(), 1), C);
652 assert(RHS && "Constant folding of immediate constants failed");
653 return Instruction::Mul;
654 }
655 // TODO: We can add other conversions e.g. shr => div etc.
656 }
657 if (Instruction::isBitwiseLogicOp(TopOpcode)) {
658 if (OtherOp && OtherOp->getOpcode() == Instruction::AShr &&
660 // lshr nneg C, X --> ashr nneg C, X
661 return Instruction::AShr;
662 }
663 }
664 return Op->getOpcode();
665}
666
667/// This tries to simplify binary operations by factorizing out common terms
668/// (e. g. "(A*B)+(A*C)" -> "A*(B+C)").
671 Instruction::BinaryOps InnerOpcode, Value *A,
672 Value *B, Value *C, Value *D) {
673 assert(A && B && C && D && "All values must be provided");
674
675 Value *V = nullptr;
676 Value *RetVal = nullptr;
677 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
678 Instruction::BinaryOps TopLevelOpcode = I.getOpcode();
679
680 // Does "X op' Y" always equal "Y op' X"?
681 bool InnerCommutative = Instruction::isCommutative(InnerOpcode);
682
683 // Does "X op' (Y op Z)" always equal "(X op' Y) op (X op' Z)"?
684 if (leftDistributesOverRight(InnerOpcode, TopLevelOpcode)) {
685 // Does the instruction have the form "(A op' B) op (A op' D)" or, in the
686 // commutative case, "(A op' B) op (C op' A)"?
687 if (A == C || (InnerCommutative && A == D)) {
688 if (A != C)
689 std::swap(C, D);
690 // Consider forming "A op' (B op D)".
691 // If "B op D" simplifies then it can be formed with no cost.
692 V = simplifyBinOp(TopLevelOpcode, B, D, SQ.getWithInstruction(&I));
693
694 // If "B op D" doesn't simplify then only go on if one of the existing
695 // operations "A op' B" and "C op' D" will be zapped as no longer used.
696 if (!V && (LHS->hasOneUse() || RHS->hasOneUse()))
697 V = Builder.CreateBinOp(TopLevelOpcode, B, D, RHS->getName());
698 if (V)
699 RetVal = Builder.CreateBinOp(InnerOpcode, A, V);
700 }
701 }
702
703 // Does "(X op Y) op' Z" always equal "(X op' Z) op (Y op' Z)"?
704 if (!RetVal && rightDistributesOverLeft(TopLevelOpcode, InnerOpcode)) {
705 // Does the instruction have the form "(A op' B) op (C op' B)" or, in the
706 // commutative case, "(A op' B) op (B op' D)"?
707 if (B == D || (InnerCommutative && B == C)) {
708 if (B != D)
709 std::swap(C, D);
710 // Consider forming "(A op C) op' B".
711 // If "A op C" simplifies then it can be formed with no cost.
712 V = simplifyBinOp(TopLevelOpcode, A, C, SQ.getWithInstruction(&I));
713
714 // If "A op C" doesn't simplify then only go on if one of the existing
715 // operations "A op' B" and "C op' D" will be zapped as no longer used.
716 if (!V && (LHS->hasOneUse() || RHS->hasOneUse()))
717 V = Builder.CreateBinOp(TopLevelOpcode, A, C, LHS->getName());
718 if (V)
719 RetVal = Builder.CreateBinOp(InnerOpcode, V, B);
720 }
721 }
722
723 if (!RetVal)
724 return nullptr;
725
726 ++NumFactor;
727 RetVal->takeName(&I);
728
729 // Try to add no-overflow flags to the final value.
730 if (isa<OverflowingBinaryOperator>(RetVal)) {
731 bool HasNSW = false;
732 bool HasNUW = false;
733 if (isa<OverflowingBinaryOperator>(&I)) {
734 HasNSW = I.hasNoSignedWrap();
735 HasNUW = I.hasNoUnsignedWrap();
736 }
737 if (auto *LOBO = dyn_cast<OverflowingBinaryOperator>(LHS)) {
738 HasNSW &= LOBO->hasNoSignedWrap();
739 HasNUW &= LOBO->hasNoUnsignedWrap();
740 }
741
742 if (auto *ROBO = dyn_cast<OverflowingBinaryOperator>(RHS)) {
743 HasNSW &= ROBO->hasNoSignedWrap();
744 HasNUW &= ROBO->hasNoUnsignedWrap();
745 }
746
747 if (TopLevelOpcode == Instruction::Add && InnerOpcode == Instruction::Mul) {
748 // We can propagate 'nsw' if we know that
749 // %Y = mul nsw i16 %X, C
750 // %Z = add nsw i16 %Y, %X
751 // =>
752 // %Z = mul nsw i16 %X, C+1
753 //
754 // iff C+1 isn't INT_MIN
755 const APInt *CInt;
756 if (match(V, m_APInt(CInt)) && !CInt->isMinSignedValue())
757 cast<Instruction>(RetVal)->setHasNoSignedWrap(HasNSW);
758
759 // nuw can be propagated with any constant or nuw value.
760 cast<Instruction>(RetVal)->setHasNoUnsignedWrap(HasNUW);
761 }
762 }
763 return RetVal;
764}
765
766// If `I` has one Const operand and the other matches `(ctpop (not x))`,
767// replace `(ctpop (not x))` with `(sub nuw nsw BitWidth(x), (ctpop x))`.
768// This is only useful is the new subtract can fold so we only handle the
769// following cases:
770// 1) (add/sub/disjoint_or C, (ctpop (not x))
771// -> (add/sub/disjoint_or C', (ctpop x))
772// 1) (cmp pred C, (ctpop (not x))
773// -> (cmp pred C', (ctpop x))
775 unsigned Opc = I->getOpcode();
776 unsigned ConstIdx = 1;
777 switch (Opc) {
778 default:
779 return nullptr;
780 // (ctpop (not x)) <-> (sub nuw nsw BitWidth(x) - (ctpop x))
781 // We can fold the BitWidth(x) with add/sub/icmp as long the other operand
782 // is constant.
783 case Instruction::Sub:
784 ConstIdx = 0;
785 break;
786 case Instruction::ICmp:
787 // Signed predicates aren't correct in some edge cases like for i2 types, as
788 // well since (ctpop x) is known [0, log2(BitWidth(x))] almost all signed
789 // comparisons against it are simplfied to unsigned.
790 if (cast<ICmpInst>(I)->isSigned())
791 return nullptr;
792 break;
793 case Instruction::Or:
794 if (!match(I, m_DisjointOr(m_Value(), m_Value())))
795 return nullptr;
796 [[fallthrough]];
797 case Instruction::Add:
798 break;
799 }
800
801 Value *Op;
802 // Find ctpop.
803 if (!match(I->getOperand(1 - ConstIdx),
804 m_OneUse(m_Intrinsic<Intrinsic::ctpop>(m_Value(Op)))))
805 return nullptr;
806
807 Constant *C;
808 // Check other operand is ImmConstant.
809 if (!match(I->getOperand(ConstIdx), m_ImmConstant(C)))
810 return nullptr;
811
812 Type *Ty = Op->getType();
813 Constant *BitWidthC = ConstantInt::get(Ty, Ty->getScalarSizeInBits());
814 // Need extra check for icmp. Note if this check is true, it generally means
815 // the icmp will simplify to true/false.
816 if (Opc == Instruction::ICmp && !cast<ICmpInst>(I)->isEquality()) {
817 Constant *Cmp =
819 if (!Cmp || !Cmp->isZeroValue())
820 return nullptr;
821 }
822
823 // Check we can invert `(not x)` for free.
824 bool Consumes = false;
825 if (!isFreeToInvert(Op, Op->hasOneUse(), Consumes) || !Consumes)
826 return nullptr;
827 Value *NotOp = getFreelyInverted(Op, Op->hasOneUse(), &Builder);
828 assert(NotOp != nullptr &&
829 "Desync between isFreeToInvert and getFreelyInverted");
830
831 Value *CtpopOfNotOp = Builder.CreateIntrinsic(Ty, Intrinsic::ctpop, NotOp);
832
833 Value *R = nullptr;
834
835 // Do the transformation here to avoid potentially introducing an infinite
836 // loop.
837 switch (Opc) {
838 case Instruction::Sub:
839 R = Builder.CreateAdd(CtpopOfNotOp, ConstantExpr::getSub(C, BitWidthC));
840 break;
841 case Instruction::Or:
842 case Instruction::Add:
843 R = Builder.CreateSub(ConstantExpr::getAdd(C, BitWidthC), CtpopOfNotOp);
844 break;
845 case Instruction::ICmp:
846 R = Builder.CreateICmp(cast<ICmpInst>(I)->getSwappedPredicate(),
847 CtpopOfNotOp, ConstantExpr::getSub(BitWidthC, C));
848 break;
849 default:
850 llvm_unreachable("Unhandled Opcode");
851 }
852 assert(R != nullptr);
853 return replaceInstUsesWith(*I, R);
854}
855
856// (Binop1 (Binop2 (logic_shift X, C), C1), (logic_shift Y, C))
857// IFF
858// 1) the logic_shifts match
859// 2) either both binops are binops and one is `and` or
860// BinOp1 is `and`
861// (logic_shift (inv_logic_shift C1, C), C) == C1 or
862//
863// -> (logic_shift (Binop1 (Binop2 X, inv_logic_shift(C1, C)), Y), C)
864//
865// (Binop1 (Binop2 (logic_shift X, Amt), Mask), (logic_shift Y, Amt))
866// IFF
867// 1) the logic_shifts match
868// 2) BinOp1 == BinOp2 (if BinOp == `add`, then also requires `shl`).
869//
870// -> (BinOp (logic_shift (BinOp X, Y)), Mask)
871//
872// (Binop1 (Binop2 (arithmetic_shift X, Amt), Mask), (arithmetic_shift Y, Amt))
873// IFF
874// 1) Binop1 is bitwise logical operator `and`, `or` or `xor`
875// 2) Binop2 is `not`
876//
877// -> (arithmetic_shift Binop1((not X), Y), Amt)
878
880 const DataLayout &DL = I.getDataLayout();
881 auto IsValidBinOpc = [](unsigned Opc) {
882 switch (Opc) {
883 default:
884 return false;
885 case Instruction::And:
886 case Instruction::Or:
887 case Instruction::Xor:
888 case Instruction::Add:
889 // Skip Sub as we only match constant masks which will canonicalize to use
890 // add.
891 return true;
892 }
893 };
894
895 // Check if we can distribute binop arbitrarily. `add` + `lshr` has extra
896 // constraints.
897 auto IsCompletelyDistributable = [](unsigned BinOpc1, unsigned BinOpc2,
898 unsigned ShOpc) {
899 assert(ShOpc != Instruction::AShr);
900 return (BinOpc1 != Instruction::Add && BinOpc2 != Instruction::Add) ||
901 ShOpc == Instruction::Shl;
902 };
903
904 auto GetInvShift = [](unsigned ShOpc) {
905 assert(ShOpc != Instruction::AShr);
906 return ShOpc == Instruction::LShr ? Instruction::Shl : Instruction::LShr;
907 };
908
909 auto CanDistributeBinops = [&](unsigned BinOpc1, unsigned BinOpc2,
910 unsigned ShOpc, Constant *CMask,
911 Constant *CShift) {
912 // If the BinOp1 is `and` we don't need to check the mask.
913 if (BinOpc1 == Instruction::And)
914 return true;
915
916 // For all other possible transfers we need complete distributable
917 // binop/shift (anything but `add` + `lshr`).
918 if (!IsCompletelyDistributable(BinOpc1, BinOpc2, ShOpc))
919 return false;
920
921 // If BinOp2 is `and`, any mask works (this only really helps for non-splat
922 // vecs, otherwise the mask will be simplified and the following check will
923 // handle it).
924 if (BinOpc2 == Instruction::And)
925 return true;
926
927 // Otherwise, need mask that meets the below requirement.
928 // (logic_shift (inv_logic_shift Mask, ShAmt), ShAmt) == Mask
929 Constant *MaskInvShift =
930 ConstantFoldBinaryOpOperands(GetInvShift(ShOpc), CMask, CShift, DL);
931 return ConstantFoldBinaryOpOperands(ShOpc, MaskInvShift, CShift, DL) ==
932 CMask;
933 };
934
935 auto MatchBinOp = [&](unsigned ShOpnum) -> Instruction * {
936 Constant *CMask, *CShift;
937 Value *X, *Y, *ShiftedX, *Mask, *Shift;
938 if (!match(I.getOperand(ShOpnum),
939 m_OneUse(m_Shift(m_Value(Y), m_Value(Shift)))))
940 return nullptr;
941 if (!match(I.getOperand(1 - ShOpnum),
942 m_BinOp(m_Value(ShiftedX), m_Value(Mask))))
943 return nullptr;
944
945 if (!match(ShiftedX, m_OneUse(m_Shift(m_Value(X), m_Specific(Shift)))))
946 return nullptr;
947
948 // Make sure we are matching instruction shifts and not ConstantExpr
949 auto *IY = dyn_cast<Instruction>(I.getOperand(ShOpnum));
950 auto *IX = dyn_cast<Instruction>(ShiftedX);
951 if (!IY || !IX)
952 return nullptr;
953
954 // LHS and RHS need same shift opcode
955 unsigned ShOpc = IY->getOpcode();
956 if (ShOpc != IX->getOpcode())
957 return nullptr;
958
959 // Make sure binop is real instruction and not ConstantExpr
960 auto *BO2 = dyn_cast<Instruction>(I.getOperand(1 - ShOpnum));
961 if (!BO2)
962 return nullptr;
963
964 unsigned BinOpc = BO2->getOpcode();
965 // Make sure we have valid binops.
966 if (!IsValidBinOpc(I.getOpcode()) || !IsValidBinOpc(BinOpc))
967 return nullptr;
968
969 if (ShOpc == Instruction::AShr) {
970 if (Instruction::isBitwiseLogicOp(I.getOpcode()) &&
971 BinOpc == Instruction::Xor && match(Mask, m_AllOnes())) {
972 Value *NotX = Builder.CreateNot(X);
973 Value *NewBinOp = Builder.CreateBinOp(I.getOpcode(), Y, NotX);
975 static_cast<Instruction::BinaryOps>(ShOpc), NewBinOp, Shift);
976 }
977
978 return nullptr;
979 }
980
981 // If BinOp1 == BinOp2 and it's bitwise or shl with add, then just
982 // distribute to drop the shift irrelevant of constants.
983 if (BinOpc == I.getOpcode() &&
984 IsCompletelyDistributable(I.getOpcode(), BinOpc, ShOpc)) {
985 Value *NewBinOp2 = Builder.CreateBinOp(I.getOpcode(), X, Y);
986 Value *NewBinOp1 = Builder.CreateBinOp(
987 static_cast<Instruction::BinaryOps>(ShOpc), NewBinOp2, Shift);
988 return BinaryOperator::Create(I.getOpcode(), NewBinOp1, Mask);
989 }
990
991 // Otherwise we can only distribute by constant shifting the mask, so
992 // ensure we have constants.
993 if (!match(Shift, m_ImmConstant(CShift)))
994 return nullptr;
995 if (!match(Mask, m_ImmConstant(CMask)))
996 return nullptr;
997
998 // Check if we can distribute the binops.
999 if (!CanDistributeBinops(I.getOpcode(), BinOpc, ShOpc, CMask, CShift))
1000 return nullptr;
1001
1002 Constant *NewCMask =
1003 ConstantFoldBinaryOpOperands(GetInvShift(ShOpc), CMask, CShift, DL);
1004 Value *NewBinOp2 = Builder.CreateBinOp(
1005 static_cast<Instruction::BinaryOps>(BinOpc), X, NewCMask);
1006 Value *NewBinOp1 = Builder.CreateBinOp(I.getOpcode(), Y, NewBinOp2);
1007 return BinaryOperator::Create(static_cast<Instruction::BinaryOps>(ShOpc),
1008 NewBinOp1, CShift);
1009 };
1010
1011 if (Instruction *R = MatchBinOp(0))
1012 return R;
1013 return MatchBinOp(1);
1014}
1015
1016// (Binop (zext C), (select C, T, F))
1017// -> (select C, (binop 1, T), (binop 0, F))
1018//
1019// (Binop (sext C), (select C, T, F))
1020// -> (select C, (binop -1, T), (binop 0, F))
1021//
1022// Attempt to simplify binary operations into a select with folded args, when
1023// one operand of the binop is a select instruction and the other operand is a
1024// zext/sext extension, whose value is the select condition.
1027 // TODO: this simplification may be extended to any speculatable instruction,
1028 // not just binops, and would possibly be handled better in FoldOpIntoSelect.
1029 Instruction::BinaryOps Opc = I.getOpcode();
1030 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
1031 Value *A, *CondVal, *TrueVal, *FalseVal;
1032 Value *CastOp;
1033
1034 auto MatchSelectAndCast = [&](Value *CastOp, Value *SelectOp) {
1035 return match(CastOp, m_ZExtOrSExt(m_Value(A))) &&
1036 A->getType()->getScalarSizeInBits() == 1 &&
1037 match(SelectOp, m_Select(m_Value(CondVal), m_Value(TrueVal),
1038 m_Value(FalseVal)));
1039 };
1040
1041 // Make sure one side of the binop is a select instruction, and the other is a
1042 // zero/sign extension operating on a i1.
1043 if (MatchSelectAndCast(LHS, RHS))
1044 CastOp = LHS;
1045 else if (MatchSelectAndCast(RHS, LHS))
1046 CastOp = RHS;
1047 else
1048 return nullptr;
1049
1050 auto NewFoldedConst = [&](bool IsTrueArm, Value *V) {
1051 bool IsCastOpRHS = (CastOp == RHS);
1052 bool IsZExt = isa<ZExtInst>(CastOp);
1053 Constant *C;
1054
1055 if (IsTrueArm) {
1056 C = Constant::getNullValue(V->getType());
1057 } else if (IsZExt) {
1058 unsigned BitWidth = V->getType()->getScalarSizeInBits();
1059 C = Constant::getIntegerValue(V->getType(), APInt(BitWidth, 1));
1060 } else {
1061 C = Constant::getAllOnesValue(V->getType());
1062 }
1063
1064 return IsCastOpRHS ? Builder.CreateBinOp(Opc, V, C)
1065 : Builder.CreateBinOp(Opc, C, V);
1066 };
1067
1068 // If the value used in the zext/sext is the select condition, or the negated
1069 // of the select condition, the binop can be simplified.
1070 if (CondVal == A) {
1071 Value *NewTrueVal = NewFoldedConst(false, TrueVal);
1072 return SelectInst::Create(CondVal, NewTrueVal,
1073 NewFoldedConst(true, FalseVal));
1074 }
1075
1076 if (match(A, m_Not(m_Specific(CondVal)))) {
1077 Value *NewTrueVal = NewFoldedConst(true, TrueVal);
1078 return SelectInst::Create(CondVal, NewTrueVal,
1079 NewFoldedConst(false, FalseVal));
1080 }
1081
1082 return nullptr;
1083}
1084
1086 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
1087 BinaryOperator *Op0 = dyn_cast<BinaryOperator>(LHS);
1088 BinaryOperator *Op1 = dyn_cast<BinaryOperator>(RHS);
1089 Instruction::BinaryOps TopLevelOpcode = I.getOpcode();
1090 Value *A, *B, *C, *D;
1091 Instruction::BinaryOps LHSOpcode, RHSOpcode;
1092
1093 if (Op0)
1094 LHSOpcode = getBinOpsForFactorization(TopLevelOpcode, Op0, A, B, Op1);
1095 if (Op1)
1096 RHSOpcode = getBinOpsForFactorization(TopLevelOpcode, Op1, C, D, Op0);
1097
1098 // The instruction has the form "(A op' B) op (C op' D)". Try to factorize
1099 // a common term.
1100 if (Op0 && Op1 && LHSOpcode == RHSOpcode)
1101 if (Value *V = tryFactorization(I, SQ, Builder, LHSOpcode, A, B, C, D))
1102 return V;
1103
1104 // The instruction has the form "(A op' B) op (C)". Try to factorize common
1105 // term.
1106 if (Op0)
1107 if (Value *Ident = getIdentityValue(LHSOpcode, RHS))
1108 if (Value *V =
1109 tryFactorization(I, SQ, Builder, LHSOpcode, A, B, RHS, Ident))
1110 return V;
1111
1112 // The instruction has the form "(B) op (C op' D)". Try to factorize common
1113 // term.
1114 if (Op1)
1115 if (Value *Ident = getIdentityValue(RHSOpcode, LHS))
1116 if (Value *V =
1117 tryFactorization(I, SQ, Builder, RHSOpcode, LHS, Ident, C, D))
1118 return V;
1119
1120 return nullptr;
1121}
1122
1123/// This tries to simplify binary operations which some other binary operation
1124/// distributes over either by factorizing out common terms
1125/// (eg "(A*B)+(A*C)" -> "A*(B+C)") or expanding out if this results in
1126/// simplifications (eg: "A & (B | C) -> (A&B) | (A&C)" if this is a win).
1127/// Returns the simplified value, or null if it didn't simplify.
1129 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
1130 BinaryOperator *Op0 = dyn_cast<BinaryOperator>(LHS);
1131 BinaryOperator *Op1 = dyn_cast<BinaryOperator>(RHS);
1132 Instruction::BinaryOps TopLevelOpcode = I.getOpcode();
1133
1134 // Factorization.
1135 if (Value *R = tryFactorizationFolds(I))
1136 return R;
1137
1138 // Expansion.
1139 if (Op0 && rightDistributesOverLeft(Op0->getOpcode(), TopLevelOpcode)) {
1140 // The instruction has the form "(A op' B) op C". See if expanding it out
1141 // to "(A op C) op' (B op C)" results in simplifications.
1142 Value *A = Op0->getOperand(0), *B = Op0->getOperand(1), *C = RHS;
1143 Instruction::BinaryOps InnerOpcode = Op0->getOpcode(); // op'
1144
1145 // Disable the use of undef because it's not safe to distribute undef.
1146 auto SQDistributive = SQ.getWithInstruction(&I).getWithoutUndef();
1147 Value *L = simplifyBinOp(TopLevelOpcode, A, C, SQDistributive);
1148 Value *R = simplifyBinOp(TopLevelOpcode, B, C, SQDistributive);
1149
1150 // Do "A op C" and "B op C" both simplify?
1151 if (L && R) {
1152 // They do! Return "L op' R".
1153 ++NumExpand;
1154 C = Builder.CreateBinOp(InnerOpcode, L, R);
1155 C->takeName(&I);
1156 return C;
1157 }
1158
1159 // Does "A op C" simplify to the identity value for the inner opcode?
1160 if (L && L == ConstantExpr::getBinOpIdentity(InnerOpcode, L->getType())) {
1161 // They do! Return "B op C".
1162 ++NumExpand;
1163 C = Builder.CreateBinOp(TopLevelOpcode, B, C);
1164 C->takeName(&I);
1165 return C;
1166 }
1167
1168 // Does "B op C" simplify to the identity value for the inner opcode?
1169 if (R && R == ConstantExpr::getBinOpIdentity(InnerOpcode, R->getType())) {
1170 // They do! Return "A op C".
1171 ++NumExpand;
1172 C = Builder.CreateBinOp(TopLevelOpcode, A, C);
1173 C->takeName(&I);
1174 return C;
1175 }
1176 }
1177
1178 if (Op1 && leftDistributesOverRight(TopLevelOpcode, Op1->getOpcode())) {
1179 // The instruction has the form "A op (B op' C)". See if expanding it out
1180 // to "(A op B) op' (A op C)" results in simplifications.
1181 Value *A = LHS, *B = Op1->getOperand(0), *C = Op1->getOperand(1);
1182 Instruction::BinaryOps InnerOpcode = Op1->getOpcode(); // op'
1183
1184 // Disable the use of undef because it's not safe to distribute undef.
1185 auto SQDistributive = SQ.getWithInstruction(&I).getWithoutUndef();
1186 Value *L = simplifyBinOp(TopLevelOpcode, A, B, SQDistributive);
1187 Value *R = simplifyBinOp(TopLevelOpcode, A, C, SQDistributive);
1188
1189 // Do "A op B" and "A op C" both simplify?
1190 if (L && R) {
1191 // They do! Return "L op' R".
1192 ++NumExpand;
1193 A = Builder.CreateBinOp(InnerOpcode, L, R);
1194 A->takeName(&I);
1195 return A;
1196 }
1197
1198 // Does "A op B" simplify to the identity value for the inner opcode?
1199 if (L && L == ConstantExpr::getBinOpIdentity(InnerOpcode, L->getType())) {
1200 // They do! Return "A op C".
1201 ++NumExpand;
1202 A = Builder.CreateBinOp(TopLevelOpcode, A, C);
1203 A->takeName(&I);
1204 return A;
1205 }
1206
1207 // Does "A op C" simplify to the identity value for the inner opcode?
1208 if (R && R == ConstantExpr::getBinOpIdentity(InnerOpcode, R->getType())) {
1209 // They do! Return "A op B".
1210 ++NumExpand;
1211 A = Builder.CreateBinOp(TopLevelOpcode, A, B);
1212 A->takeName(&I);
1213 return A;
1214 }
1215 }
1216
1218}
1219
1220static std::optional<std::pair<Value *, Value *>>
1222 if (LHS->getParent() != RHS->getParent())
1223 return std::nullopt;
1224
1225 if (LHS->getNumIncomingValues() < 2)
1226 return std::nullopt;
1227
1228 if (!equal(LHS->blocks(), RHS->blocks()))
1229 return std::nullopt;
1230
1231 Value *L0 = LHS->getIncomingValue(0);
1232 Value *R0 = RHS->getIncomingValue(0);
1233
1234 for (unsigned I = 1, E = LHS->getNumIncomingValues(); I != E; ++I) {
1235 Value *L1 = LHS->getIncomingValue(I);
1236 Value *R1 = RHS->getIncomingValue(I);
1237
1238 if ((L0 == L1 && R0 == R1) || (L0 == R1 && R0 == L1))
1239 continue;
1240
1241 return std::nullopt;
1242 }
1243
1244 return std::optional(std::pair(L0, R0));
1245}
1246
1247std::optional<std::pair<Value *, Value *>>
1248InstCombinerImpl::matchSymmetricPair(Value *LHS, Value *RHS) {
1249 Instruction *LHSInst = dyn_cast<Instruction>(LHS);
1250 Instruction *RHSInst = dyn_cast<Instruction>(RHS);
1251 if (!LHSInst || !RHSInst || LHSInst->getOpcode() != RHSInst->getOpcode())
1252 return std::nullopt;
1253 switch (LHSInst->getOpcode()) {
1254 case Instruction::PHI:
1255 return matchSymmetricPhiNodesPair(cast<PHINode>(LHS), cast<PHINode>(RHS));
1256 case Instruction::Select: {
1257 Value *Cond = LHSInst->getOperand(0);
1258 Value *TrueVal = LHSInst->getOperand(1);
1259 Value *FalseVal = LHSInst->getOperand(2);
1260 if (Cond == RHSInst->getOperand(0) && TrueVal == RHSInst->getOperand(2) &&
1261 FalseVal == RHSInst->getOperand(1))
1262 return std::pair(TrueVal, FalseVal);
1263 return std::nullopt;
1264 }
1265 case Instruction::Call: {
1266 // Match min(a, b) and max(a, b)
1267 MinMaxIntrinsic *LHSMinMax = dyn_cast<MinMaxIntrinsic>(LHSInst);
1268 MinMaxIntrinsic *RHSMinMax = dyn_cast<MinMaxIntrinsic>(RHSInst);
1269 if (LHSMinMax && RHSMinMax &&
1270 LHSMinMax->getPredicate() ==
1272 ((LHSMinMax->getLHS() == RHSMinMax->getLHS() &&
1273 LHSMinMax->getRHS() == RHSMinMax->getRHS()) ||
1274 (LHSMinMax->getLHS() == RHSMinMax->getRHS() &&
1275 LHSMinMax->getRHS() == RHSMinMax->getLHS())))
1276 return std::pair(LHSMinMax->getLHS(), LHSMinMax->getRHS());
1277 return std::nullopt;
1278 }
1279 default:
1280 return std::nullopt;
1281 }
1282}
1283
1285 Value *LHS,
1286 Value *RHS) {
1287 Value *A, *B, *C, *D, *E, *F;
1288 bool LHSIsSelect = match(LHS, m_Select(m_Value(A), m_Value(B), m_Value(C)));
1289 bool RHSIsSelect = match(RHS, m_Select(m_Value(D), m_Value(E), m_Value(F)));
1290 if (!LHSIsSelect && !RHSIsSelect)
1291 return nullptr;
1292
1293 FastMathFlags FMF;
1295 if (isa<FPMathOperator>(&I)) {
1296 FMF = I.getFastMathFlags();
1298 }
1299
1300 Instruction::BinaryOps Opcode = I.getOpcode();
1302
1303 Value *Cond, *True = nullptr, *False = nullptr;
1304
1305 // Special-case for add/negate combination. Replace the zero in the negation
1306 // with the trailing add operand:
1307 // (Cond ? TVal : -N) + Z --> Cond ? True : (Z - N)
1308 // (Cond ? -N : FVal) + Z --> Cond ? (Z - N) : False
1309 auto foldAddNegate = [&](Value *TVal, Value *FVal, Value *Z) -> Value * {
1310 // We need an 'add' and exactly 1 arm of the select to have been simplified.
1311 if (Opcode != Instruction::Add || (!True && !False) || (True && False))
1312 return nullptr;
1313
1314 Value *N;
1315 if (True && match(FVal, m_Neg(m_Value(N)))) {
1316 Value *Sub = Builder.CreateSub(Z, N);
1317 return Builder.CreateSelect(Cond, True, Sub, I.getName());
1318 }
1319 if (False && match(TVal, m_Neg(m_Value(N)))) {
1320 Value *Sub = Builder.CreateSub(Z, N);
1321 return Builder.CreateSelect(Cond, Sub, False, I.getName());
1322 }
1323 return nullptr;
1324 };
1325
1326 if (LHSIsSelect && RHSIsSelect && A == D) {
1327 // (A ? B : C) op (A ? E : F) -> A ? (B op E) : (C op F)
1328 Cond = A;
1329 True = simplifyBinOp(Opcode, B, E, FMF, Q);
1330 False = simplifyBinOp(Opcode, C, F, FMF, Q);
1331
1332 if (LHS->hasOneUse() && RHS->hasOneUse()) {
1333 if (False && !True)
1334 True = Builder.CreateBinOp(Opcode, B, E);
1335 else if (True && !False)
1336 False = Builder.CreateBinOp(Opcode, C, F);
1337 }
1338 } else if (LHSIsSelect && LHS->hasOneUse()) {
1339 // (A ? B : C) op Y -> A ? (B op Y) : (C op Y)
1340 Cond = A;
1341 True = simplifyBinOp(Opcode, B, RHS, FMF, Q);
1342 False = simplifyBinOp(Opcode, C, RHS, FMF, Q);
1343 if (Value *NewSel = foldAddNegate(B, C, RHS))
1344 return NewSel;
1345 } else if (RHSIsSelect && RHS->hasOneUse()) {
1346 // X op (D ? E : F) -> D ? (X op E) : (X op F)
1347 Cond = D;
1348 True = simplifyBinOp(Opcode, LHS, E, FMF, Q);
1349 False = simplifyBinOp(Opcode, LHS, F, FMF, Q);
1350 if (Value *NewSel = foldAddNegate(E, F, LHS))
1351 return NewSel;
1352 }
1353
1354 if (!True || !False)
1355 return nullptr;
1356
1357 Value *SI = Builder.CreateSelect(Cond, True, False);
1358 SI->takeName(&I);
1359 return SI;
1360}
1361
1362/// Freely adapt every user of V as-if V was changed to !V.
1363/// WARNING: only if canFreelyInvertAllUsersOf() said this can be done.
1365 assert(!isa<Constant>(I) && "Shouldn't invert users of constant");
1366 for (User *U : make_early_inc_range(I->users())) {
1367 if (U == IgnoredUser)
1368 continue; // Don't consider this user.
1369 switch (cast<Instruction>(U)->getOpcode()) {
1370 case Instruction::Select: {
1371 auto *SI = cast<SelectInst>(U);
1372 SI->swapValues();
1373 SI->swapProfMetadata();
1374 break;
1375 }
1376 case Instruction::Br: {
1377 BranchInst *BI = cast<BranchInst>(U);
1378 BI->swapSuccessors(); // swaps prof metadata too
1379 if (BPI)
1381 break;
1382 }
1383 case Instruction::Xor:
1384 replaceInstUsesWith(cast<Instruction>(*U), I);
1385 // Add to worklist for DCE.
1386 addToWorklist(cast<Instruction>(U));
1387 break;
1388 default:
1389 llvm_unreachable("Got unexpected user - out of sync with "
1390 "canFreelyInvertAllUsersOf() ?");
1391 }
1392 }
1393}
1394
1395/// Given a 'sub' instruction, return the RHS of the instruction if the LHS is a
1396/// constant zero (which is the 'negate' form).
1397Value *InstCombinerImpl::dyn_castNegVal(Value *V) const {
1398 Value *NegV;
1399 if (match(V, m_Neg(m_Value(NegV))))
1400 return NegV;
1401
1402 // Constants can be considered to be negated values if they can be folded.
1403 if (ConstantInt *C = dyn_cast<ConstantInt>(V))
1404 return ConstantExpr::getNeg(C);
1405
1406 if (ConstantDataVector *C = dyn_cast<ConstantDataVector>(V))
1407 if (C->getType()->getElementType()->isIntegerTy())
1408 return ConstantExpr::getNeg(C);
1409
1410 if (ConstantVector *CV = dyn_cast<ConstantVector>(V)) {
1411 for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
1412 Constant *Elt = CV->getAggregateElement(i);
1413 if (!Elt)
1414 return nullptr;
1415
1416 if (isa<UndefValue>(Elt))
1417 continue;
1418
1419 if (!isa<ConstantInt>(Elt))
1420 return nullptr;
1421 }
1422 return ConstantExpr::getNeg(CV);
1423 }
1424
1425 // Negate integer vector splats.
1426 if (auto *CV = dyn_cast<Constant>(V))
1427 if (CV->getType()->isVectorTy() &&
1428 CV->getType()->getScalarType()->isIntegerTy() && CV->getSplatValue())
1429 return ConstantExpr::getNeg(CV);
1430
1431 return nullptr;
1432}
1433
1434// Try to fold:
1435// 1) (fp_binop ({s|u}itofp x), ({s|u}itofp y))
1436// -> ({s|u}itofp (int_binop x, y))
1437// 2) (fp_binop ({s|u}itofp x), FpC)
1438// -> ({s|u}itofp (int_binop x, (fpto{s|u}i FpC)))
1439//
1440// Assuming the sign of the cast for x/y is `OpsFromSigned`.
1441Instruction *InstCombinerImpl::foldFBinOpOfIntCastsFromSign(
1442 BinaryOperator &BO, bool OpsFromSigned, std::array<Value *, 2> IntOps,
1444
1445 Type *FPTy = BO.getType();
1446 Type *IntTy = IntOps[0]->getType();
1447
1448 unsigned IntSz = IntTy->getScalarSizeInBits();
1449 // This is the maximum number of inuse bits by the integer where the int -> fp
1450 // casts are exact.
1451 unsigned MaxRepresentableBits =
1453
1454 // Preserve known number of leading bits. This can allow us to trivial nsw/nuw
1455 // checks later on.
1456 unsigned NumUsedLeadingBits[2] = {IntSz, IntSz};
1457
1458 // NB: This only comes up if OpsFromSigned is true, so there is no need to
1459 // cache if between calls to `foldFBinOpOfIntCastsFromSign`.
1460 auto IsNonZero = [&](unsigned OpNo) -> bool {
1461 if (OpsKnown[OpNo].hasKnownBits() &&
1462 OpsKnown[OpNo].getKnownBits(SQ).isNonZero())
1463 return true;
1464 return isKnownNonZero(IntOps[OpNo], SQ);
1465 };
1466
1467 auto IsNonNeg = [&](unsigned OpNo) -> bool {
1468 // NB: This matches the impl in ValueTracking, we just try to use cached
1469 // knownbits here. If we ever start supporting WithCache for
1470 // `isKnownNonNegative`, change this to an explicit call.
1471 return OpsKnown[OpNo].getKnownBits(SQ).isNonNegative();
1472 };
1473
1474 // Check if we know for certain that ({s|u}itofp op) is exact.
1475 auto IsValidPromotion = [&](unsigned OpNo) -> bool {
1476 // Can we treat this operand as the desired sign?
1477 if (OpsFromSigned != isa<SIToFPInst>(BO.getOperand(OpNo)) &&
1478 !IsNonNeg(OpNo))
1479 return false;
1480
1481 // If fp precision >= bitwidth(op) then its exact.
1482 // NB: This is slightly conservative for `sitofp`. For signed conversion, we
1483 // can handle `MaxRepresentableBits == IntSz - 1` as the sign bit will be
1484 // handled specially. We can't, however, increase the bound arbitrarily for
1485 // `sitofp` as for larger sizes, it won't sign extend.
1486 if (MaxRepresentableBits < IntSz) {
1487 // Otherwise if its signed cast check that fp precisions >= bitwidth(op) -
1488 // numSignBits(op).
1489 // TODO: If we add support for `WithCache` in `ComputeNumSignBits`, change
1490 // `IntOps[OpNo]` arguments to `KnownOps[OpNo]`.
1491 if (OpsFromSigned)
1492 NumUsedLeadingBits[OpNo] = IntSz - ComputeNumSignBits(IntOps[OpNo]);
1493 // Finally for unsigned check that fp precision >= bitwidth(op) -
1494 // numLeadingZeros(op).
1495 else {
1496 NumUsedLeadingBits[OpNo] =
1497 IntSz - OpsKnown[OpNo].getKnownBits(SQ).countMinLeadingZeros();
1498 }
1499 }
1500 // NB: We could also check if op is known to be a power of 2 or zero (which
1501 // will always be representable). Its unlikely, however, that is we are
1502 // unable to bound op in any way we will be able to pass the overflow checks
1503 // later on.
1504
1505 if (MaxRepresentableBits < NumUsedLeadingBits[OpNo])
1506 return false;
1507 // Signed + Mul also requires that op is non-zero to avoid -0 cases.
1508 return !OpsFromSigned || BO.getOpcode() != Instruction::FMul ||
1509 IsNonZero(OpNo);
1510 };
1511
1512 // If we have a constant rhs, see if we can losslessly convert it to an int.
1513 if (Op1FpC != nullptr) {
1514 // Signed + Mul req non-zero
1515 if (OpsFromSigned && BO.getOpcode() == Instruction::FMul &&
1516 !match(Op1FpC, m_NonZeroFP()))
1517 return nullptr;
1518
1520 OpsFromSigned ? Instruction::FPToSI : Instruction::FPToUI, Op1FpC,
1521 IntTy, DL);
1522 if (Op1IntC == nullptr)
1523 return nullptr;
1524 if (ConstantFoldCastOperand(OpsFromSigned ? Instruction::SIToFP
1525 : Instruction::UIToFP,
1526 Op1IntC, FPTy, DL) != Op1FpC)
1527 return nullptr;
1528
1529 // First try to keep sign of cast the same.
1530 IntOps[1] = Op1IntC;
1531 }
1532
1533 // Ensure lhs/rhs integer types match.
1534 if (IntTy != IntOps[1]->getType())
1535 return nullptr;
1536
1537 if (Op1FpC == nullptr) {
1538 if (!IsValidPromotion(1))
1539 return nullptr;
1540 }
1541 if (!IsValidPromotion(0))
1542 return nullptr;
1543
1544 // Final we check if the integer version of the binop will not overflow.
1546 // Because of the precision check, we can often rule out overflows.
1547 bool NeedsOverflowCheck = true;
1548 // Try to conservatively rule out overflow based on the already done precision
1549 // checks.
1550 unsigned OverflowMaxOutputBits = OpsFromSigned ? 2 : 1;
1551 unsigned OverflowMaxCurBits =
1552 std::max(NumUsedLeadingBits[0], NumUsedLeadingBits[1]);
1553 bool OutputSigned = OpsFromSigned;
1554 switch (BO.getOpcode()) {
1555 case Instruction::FAdd:
1556 IntOpc = Instruction::Add;
1557 OverflowMaxOutputBits += OverflowMaxCurBits;
1558 break;
1559 case Instruction::FSub:
1560 IntOpc = Instruction::Sub;
1561 OverflowMaxOutputBits += OverflowMaxCurBits;
1562 break;
1563 case Instruction::FMul:
1564 IntOpc = Instruction::Mul;
1565 OverflowMaxOutputBits += OverflowMaxCurBits * 2;
1566 break;
1567 default:
1568 llvm_unreachable("Unsupported binop");
1569 }
1570 // The precision check may have already ruled out overflow.
1571 if (OverflowMaxOutputBits < IntSz) {
1572 NeedsOverflowCheck = false;
1573 // We can bound unsigned overflow from sub to in range signed value (this is
1574 // what allows us to avoid the overflow check for sub).
1575 if (IntOpc == Instruction::Sub)
1576 OutputSigned = true;
1577 }
1578
1579 // Precision check did not rule out overflow, so need to check.
1580 // TODO: If we add support for `WithCache` in `willNotOverflow`, change
1581 // `IntOps[...]` arguments to `KnownOps[...]`.
1582 if (NeedsOverflowCheck &&
1583 !willNotOverflow(IntOpc, IntOps[0], IntOps[1], BO, OutputSigned))
1584 return nullptr;
1585
1586 Value *IntBinOp = Builder.CreateBinOp(IntOpc, IntOps[0], IntOps[1]);
1587 if (auto *IntBO = dyn_cast<BinaryOperator>(IntBinOp)) {
1588 IntBO->setHasNoSignedWrap(OutputSigned);
1589 IntBO->setHasNoUnsignedWrap(!OutputSigned);
1590 }
1591 if (OutputSigned)
1592 return new SIToFPInst(IntBinOp, FPTy);
1593 return new UIToFPInst(IntBinOp, FPTy);
1594}
1595
1596// Try to fold:
1597// 1) (fp_binop ({s|u}itofp x), ({s|u}itofp y))
1598// -> ({s|u}itofp (int_binop x, y))
1599// 2) (fp_binop ({s|u}itofp x), FpC)
1600// -> ({s|u}itofp (int_binop x, (fpto{s|u}i FpC)))
1601Instruction *InstCombinerImpl::foldFBinOpOfIntCasts(BinaryOperator &BO) {
1602 std::array<Value *, 2> IntOps = {nullptr, nullptr};
1603 Constant *Op1FpC = nullptr;
1604 // Check for:
1605 // 1) (binop ({s|u}itofp x), ({s|u}itofp y))
1606 // 2) (binop ({s|u}itofp x), FpC)
1607 if (!match(BO.getOperand(0), m_SIToFP(m_Value(IntOps[0]))) &&
1608 !match(BO.getOperand(0), m_UIToFP(m_Value(IntOps[0]))))
1609 return nullptr;
1610
1611 if (!match(BO.getOperand(1), m_Constant(Op1FpC)) &&
1612 !match(BO.getOperand(1), m_SIToFP(m_Value(IntOps[1]))) &&
1613 !match(BO.getOperand(1), m_UIToFP(m_Value(IntOps[1]))))
1614 return nullptr;
1615
1616 // Cache KnownBits a bit to potentially save some analysis.
1617 SmallVector<WithCache<const Value *>, 2> OpsKnown = {IntOps[0], IntOps[1]};
1618
1619 // Try treating x/y as coming from both `uitofp` and `sitofp`. There are
1620 // different constraints depending on the sign of the cast.
1621 // NB: `(uitofp nneg X)` == `(sitofp nneg X)`.
1622 if (Instruction *R = foldFBinOpOfIntCastsFromSign(BO, /*OpsFromSigned=*/false,
1623 IntOps, Op1FpC, OpsKnown))
1624 return R;
1625 return foldFBinOpOfIntCastsFromSign(BO, /*OpsFromSigned=*/true, IntOps,
1626 Op1FpC, OpsKnown);
1627}
1628
1629/// A binop with a constant operand and a sign-extended boolean operand may be
1630/// converted into a select of constants by applying the binary operation to
1631/// the constant with the two possible values of the extended boolean (0 or -1).
1632Instruction *InstCombinerImpl::foldBinopOfSextBoolToSelect(BinaryOperator &BO) {
1633 // TODO: Handle non-commutative binop (constant is operand 0).
1634 // TODO: Handle zext.
1635 // TODO: Peek through 'not' of cast.
1636 Value *BO0 = BO.getOperand(0);
1637 Value *BO1 = BO.getOperand(1);
1638 Value *X;
1639 Constant *C;
1640 if (!match(BO0, m_SExt(m_Value(X))) || !match(BO1, m_ImmConstant(C)) ||
1641 !X->getType()->isIntOrIntVectorTy(1))
1642 return nullptr;
1643
1644 // bo (sext i1 X), C --> select X, (bo -1, C), (bo 0, C)
1647 Value *TVal = Builder.CreateBinOp(BO.getOpcode(), Ones, C);
1648 Value *FVal = Builder.CreateBinOp(BO.getOpcode(), Zero, C);
1649 return SelectInst::Create(X, TVal, FVal);
1650}
1651
1653 bool IsTrueArm) {
1655 for (Value *Op : I.operands()) {
1656 Value *V = nullptr;
1657 if (Op == SI) {
1658 V = IsTrueArm ? SI->getTrueValue() : SI->getFalseValue();
1659 } else if (match(SI->getCondition(),
1662 m_Specific(Op), m_Value(V))) &&
1664 // Pass
1665 } else {
1666 V = Op;
1667 }
1668 Ops.push_back(V);
1669 }
1670
1671 return simplifyInstructionWithOperands(&I, Ops, I.getDataLayout());
1672}
1673
1675 Value *NewOp, InstCombiner &IC) {
1676 Instruction *Clone = I.clone();
1677 Clone->replaceUsesOfWith(SI, NewOp);
1679 IC.InsertNewInstBefore(Clone, I.getIterator());
1680 return Clone;
1681}
1682
1684 bool FoldWithMultiUse) {
1685 // Don't modify shared select instructions unless set FoldWithMultiUse
1686 if (!SI->hasOneUse() && !FoldWithMultiUse)
1687 return nullptr;
1688
1689 Value *TV = SI->getTrueValue();
1690 Value *FV = SI->getFalseValue();
1691
1692 // Bool selects with constant operands can be folded to logical ops.
1693 if (SI->getType()->isIntOrIntVectorTy(1))
1694 return nullptr;
1695
1696 // Test if a FCmpInst instruction is used exclusively by a select as
1697 // part of a minimum or maximum operation. If so, refrain from doing
1698 // any other folding. This helps out other analyses which understand
1699 // non-obfuscated minimum and maximum idioms. And in this case, at
1700 // least one of the comparison operands has at least one user besides
1701 // the compare (the select), which would often largely negate the
1702 // benefit of folding anyway.
1703 if (auto *CI = dyn_cast<FCmpInst>(SI->getCondition())) {
1704 if (CI->hasOneUse()) {
1705 Value *Op0 = CI->getOperand(0), *Op1 = CI->getOperand(1);
1706 if ((TV == Op0 && FV == Op1) || (FV == Op0 && TV == Op1))
1707 return nullptr;
1708 }
1709 }
1710
1711 // Make sure that one of the select arms folds successfully.
1712 Value *NewTV = simplifyOperationIntoSelectOperand(Op, SI, /*IsTrueArm=*/true);
1713 Value *NewFV =
1714 simplifyOperationIntoSelectOperand(Op, SI, /*IsTrueArm=*/false);
1715 if (!NewTV && !NewFV)
1716 return nullptr;
1717
1718 // Create an instruction for the arm that did not fold.
1719 if (!NewTV)
1720 NewTV = foldOperationIntoSelectOperand(Op, SI, TV, *this);
1721 if (!NewFV)
1722 NewFV = foldOperationIntoSelectOperand(Op, SI, FV, *this);
1723 return SelectInst::Create(SI->getCondition(), NewTV, NewFV, "", nullptr, SI);
1724}
1725
1727 Value *InValue, BasicBlock *InBB,
1728 const DataLayout &DL,
1729 const SimplifyQuery SQ) {
1730 // NB: It is a precondition of this transform that the operands be
1731 // phi translatable!
1733 for (Value *Op : I.operands()) {
1734 if (Op == PN)
1735 Ops.push_back(InValue);
1736 else
1737 Ops.push_back(Op->DoPHITranslation(PN->getParent(), InBB));
1738 }
1739
1740 // Don't consider the simplification successful if we get back a constant
1741 // expression. That's just an instruction in hiding.
1742 // Also reject the case where we simplify back to the phi node. We wouldn't
1743 // be able to remove it in that case.
1745 &I, Ops, SQ.getWithInstruction(InBB->getTerminator()));
1746 if (NewVal && NewVal != PN && !match(NewVal, m_ConstantExpr()))
1747 return NewVal;
1748
1749 // Check if incoming PHI value can be replaced with constant
1750 // based on implied condition.
1751 BranchInst *TerminatorBI = dyn_cast<BranchInst>(InBB->getTerminator());
1752 const ICmpInst *ICmp = dyn_cast<ICmpInst>(&I);
1753 if (TerminatorBI && TerminatorBI->isConditional() &&
1754 TerminatorBI->getSuccessor(0) != TerminatorBI->getSuccessor(1) && ICmp) {
1755 bool LHSIsTrue = TerminatorBI->getSuccessor(0) == PN->getParent();
1756 std::optional<bool> ImpliedCond = isImpliedCondition(
1757 TerminatorBI->getCondition(), ICmp->getCmpPredicate(), Ops[0], Ops[1],
1758 DL, LHSIsTrue);
1759 if (ImpliedCond)
1760 return ConstantInt::getBool(I.getType(), ImpliedCond.value());
1761 }
1762
1763 return nullptr;
1764}
1765
1767 bool AllowMultipleUses) {
1768 unsigned NumPHIValues = PN->getNumIncomingValues();
1769 if (NumPHIValues == 0)
1770 return nullptr;
1771
1772 // We normally only transform phis with a single use. However, if a PHI has
1773 // multiple uses and they are all the same operation, we can fold *all* of the
1774 // uses into the PHI.
1775 bool OneUse = PN->hasOneUse();
1776 bool IdenticalUsers = false;
1777 if (!AllowMultipleUses && !OneUse) {
1778 // Walk the use list for the instruction, comparing them to I.
1779 for (User *U : PN->users()) {
1780 Instruction *UI = cast<Instruction>(U);
1781 if (UI != &I && !I.isIdenticalTo(UI))
1782 return nullptr;
1783 }
1784 // Otherwise, we can replace *all* users with the new PHI we form.
1785 IdenticalUsers = true;
1786 }
1787
1788 // Check that all operands are phi-translatable.
1789 for (Value *Op : I.operands()) {
1790 if (Op == PN)
1791 continue;
1792
1793 // Non-instructions never require phi-translation.
1794 auto *I = dyn_cast<Instruction>(Op);
1795 if (!I)
1796 continue;
1797
1798 // Phi-translate can handle phi nodes in the same block.
1799 if (isa<PHINode>(I))
1800 if (I->getParent() == PN->getParent())
1801 continue;
1802
1803 // Operand dominates the block, no phi-translation necessary.
1804 if (DT.dominates(I, PN->getParent()))
1805 continue;
1806
1807 // Not phi-translatable, bail out.
1808 return nullptr;
1809 }
1810
1811 // Check to see whether the instruction can be folded into each phi operand.
1812 // If there is one operand that does not fold, remember the BB it is in.
1813 SmallVector<Value *> NewPhiValues;
1814 SmallVector<unsigned int> OpsToMoveUseToIncomingBB;
1815 bool SeenNonSimplifiedInVal = false;
1816 for (unsigned i = 0; i != NumPHIValues; ++i) {
1817 Value *InVal = PN->getIncomingValue(i);
1818 BasicBlock *InBB = PN->getIncomingBlock(i);
1819
1820 if (auto *NewVal = simplifyInstructionWithPHI(I, PN, InVal, InBB, DL, SQ)) {
1821 NewPhiValues.push_back(NewVal);
1822 continue;
1823 }
1824
1825 // If the only use of phi is comparing it with a constant then we can
1826 // put this comparison in the incoming BB directly after a ucmp/scmp call
1827 // because we know that it will simplify to a single icmp.
1828 const APInt *Ignored;
1829 if (isa<CmpIntrinsic>(InVal) && InVal->hasOneUser() &&
1830 match(&I, m_ICmp(m_Specific(PN), m_APInt(Ignored)))) {
1831 OpsToMoveUseToIncomingBB.push_back(i);
1832 NewPhiValues.push_back(nullptr);
1833 continue;
1834 }
1835
1836 if (!OneUse && !IdenticalUsers)
1837 return nullptr;
1838
1839 if (SeenNonSimplifiedInVal)
1840 return nullptr; // More than one non-simplified value.
1841 SeenNonSimplifiedInVal = true;
1842
1843 // If there is exactly one non-simplified value, we can insert a copy of the
1844 // operation in that block. However, if this is a critical edge, we would
1845 // be inserting the computation on some other paths (e.g. inside a loop).
1846 // Only do this if the pred block is unconditionally branching into the phi
1847 // block. Also, make sure that the pred block is not dead code.
1848 BranchInst *BI = dyn_cast<BranchInst>(InBB->getTerminator());
1849 if (!BI || !BI->isUnconditional() || !DT.isReachableFromEntry(InBB))
1850 return nullptr;
1851
1852 NewPhiValues.push_back(nullptr);
1853 OpsToMoveUseToIncomingBB.push_back(i);
1854
1855 // If the InVal is an invoke at the end of the pred block, then we can't
1856 // insert a computation after it without breaking the edge.
1857 if (isa<InvokeInst>(InVal))
1858 if (cast<Instruction>(InVal)->getParent() == InBB)
1859 return nullptr;
1860
1861 // Do not push the operation across a loop backedge. This could result in
1862 // an infinite combine loop, and is generally non-profitable (especially
1863 // if the operation was originally outside the loop).
1864 if (isBackEdge(InBB, PN->getParent()))
1865 return nullptr;
1866 }
1867
1868 // Clone the instruction that uses the phi node and move it into the incoming
1869 // BB because we know that the next iteration of InstCombine will simplify it.
1871 for (auto OpIndex : OpsToMoveUseToIncomingBB) {
1873 BasicBlock *OpBB = PN->getIncomingBlock(OpIndex);
1874
1875 Instruction *Clone = Clones.lookup(OpBB);
1876 if (!Clone) {
1877 Clone = I.clone();
1878 for (Use &U : Clone->operands()) {
1879 if (U == PN)
1880 U = Op;
1881 else
1882 U = U->DoPHITranslation(PN->getParent(), OpBB);
1883 }
1884 Clone = InsertNewInstBefore(Clone, OpBB->getTerminator()->getIterator());
1885 Clones.insert({OpBB, Clone});
1886 }
1887
1888 NewPhiValues[OpIndex] = Clone;
1889 }
1890
1891 // Okay, we can do the transformation: create the new PHI node.
1892 PHINode *NewPN = PHINode::Create(I.getType(), PN->getNumIncomingValues());
1893 InsertNewInstBefore(NewPN, PN->getIterator());
1894 NewPN->takeName(PN);
1895 NewPN->setDebugLoc(PN->getDebugLoc());
1896
1897 for (unsigned i = 0; i != NumPHIValues; ++i)
1898 NewPN->addIncoming(NewPhiValues[i], PN->getIncomingBlock(i));
1899
1900 if (IdenticalUsers) {
1901 for (User *U : make_early_inc_range(PN->users())) {
1902 Instruction *User = cast<Instruction>(U);
1903 if (User == &I)
1904 continue;
1905 replaceInstUsesWith(*User, NewPN);
1907 }
1908 OneUse = true;
1909 }
1910
1911 if (OneUse) {
1912 replaceAllDbgUsesWith(const_cast<PHINode &>(*PN),
1913 const_cast<PHINode &>(*NewPN),
1914 const_cast<PHINode &>(*PN), DT);
1915 }
1916 return replaceInstUsesWith(I, NewPN);
1917}
1918
1920 // TODO: This should be similar to the incoming values check in foldOpIntoPhi:
1921 // we are guarding against replicating the binop in >1 predecessor.
1922 // This could miss matching a phi with 2 constant incoming values.
1923 auto *Phi0 = dyn_cast<PHINode>(BO.getOperand(0));
1924 auto *Phi1 = dyn_cast<PHINode>(BO.getOperand(1));
1925 if (!Phi0 || !Phi1 || !Phi0->hasOneUse() || !Phi1->hasOneUse() ||
1926 Phi0->getNumOperands() != Phi1->getNumOperands())
1927 return nullptr;
1928
1929 // TODO: Remove the restriction for binop being in the same block as the phis.
1930 if (BO.getParent() != Phi0->getParent() ||
1931 BO.getParent() != Phi1->getParent())
1932 return nullptr;
1933
1934 // Fold if there is at least one specific constant value in phi0 or phi1's
1935 // incoming values that comes from the same block and this specific constant
1936 // value can be used to do optimization for specific binary operator.
1937 // For example:
1938 // %phi0 = phi i32 [0, %bb0], [%i, %bb1]
1939 // %phi1 = phi i32 [%j, %bb0], [0, %bb1]
1940 // %add = add i32 %phi0, %phi1
1941 // ==>
1942 // %add = phi i32 [%j, %bb0], [%i, %bb1]
1944 /*AllowRHSConstant*/ false);
1945 if (C) {
1946 SmallVector<Value *, 4> NewIncomingValues;
1947 auto CanFoldIncomingValuePair = [&](std::tuple<Use &, Use &> T) {
1948 auto &Phi0Use = std::get<0>(T);
1949 auto &Phi1Use = std::get<1>(T);
1950 if (Phi0->getIncomingBlock(Phi0Use) != Phi1->getIncomingBlock(Phi1Use))
1951 return false;
1952 Value *Phi0UseV = Phi0Use.get();
1953 Value *Phi1UseV = Phi1Use.get();
1954 if (Phi0UseV == C)
1955 NewIncomingValues.push_back(Phi1UseV);
1956 else if (Phi1UseV == C)
1957 NewIncomingValues.push_back(Phi0UseV);
1958 else
1959 return false;
1960 return true;
1961 };
1962
1963 if (all_of(zip(Phi0->operands(), Phi1->operands()),
1964 CanFoldIncomingValuePair)) {
1965 PHINode *NewPhi =
1966 PHINode::Create(Phi0->getType(), Phi0->getNumOperands());
1967 assert(NewIncomingValues.size() == Phi0->getNumOperands() &&
1968 "The number of collected incoming values should equal the number "
1969 "of the original PHINode operands!");
1970 for (unsigned I = 0; I < Phi0->getNumOperands(); I++)
1971 NewPhi->addIncoming(NewIncomingValues[I], Phi0->getIncomingBlock(I));
1972 return NewPhi;
1973 }
1974 }
1975
1976 if (Phi0->getNumOperands() != 2 || Phi1->getNumOperands() != 2)
1977 return nullptr;
1978
1979 // Match a pair of incoming constants for one of the predecessor blocks.
1980 BasicBlock *ConstBB, *OtherBB;
1981 Constant *C0, *C1;
1982 if (match(Phi0->getIncomingValue(0), m_ImmConstant(C0))) {
1983 ConstBB = Phi0->getIncomingBlock(0);
1984 OtherBB = Phi0->getIncomingBlock(1);
1985 } else if (match(Phi0->getIncomingValue(1), m_ImmConstant(C0))) {
1986 ConstBB = Phi0->getIncomingBlock(1);
1987 OtherBB = Phi0->getIncomingBlock(0);
1988 } else {
1989 return nullptr;
1990 }
1991 if (!match(Phi1->getIncomingValueForBlock(ConstBB), m_ImmConstant(C1)))
1992 return nullptr;
1993
1994 // The block that we are hoisting to must reach here unconditionally.
1995 // Otherwise, we could be speculatively executing an expensive or
1996 // non-speculative op.
1997 auto *PredBlockBranch = dyn_cast<BranchInst>(OtherBB->getTerminator());
1998 if (!PredBlockBranch || PredBlockBranch->isConditional() ||
1999 !DT.isReachableFromEntry(OtherBB))
2000 return nullptr;
2001
2002 // TODO: This check could be tightened to only apply to binops (div/rem) that
2003 // are not safe to speculatively execute. But that could allow hoisting
2004 // potentially expensive instructions (fdiv for example).
2005 for (auto BBIter = BO.getParent()->begin(); &*BBIter != &BO; ++BBIter)
2007 return nullptr;
2008
2009 // Fold constants for the predecessor block with constant incoming values.
2010 Constant *NewC = ConstantFoldBinaryOpOperands(BO.getOpcode(), C0, C1, DL);
2011 if (!NewC)
2012 return nullptr;
2013
2014 // Make a new binop in the predecessor block with the non-constant incoming
2015 // values.
2016 Builder.SetInsertPoint(PredBlockBranch);
2017 Value *NewBO = Builder.CreateBinOp(BO.getOpcode(),
2018 Phi0->getIncomingValueForBlock(OtherBB),
2019 Phi1->getIncomingValueForBlock(OtherBB));
2020 if (auto *NotFoldedNewBO = dyn_cast<BinaryOperator>(NewBO))
2021 NotFoldedNewBO->copyIRFlags(&BO);
2022
2023 // Replace the binop with a phi of the new values. The old phis are dead.
2024 PHINode *NewPhi = PHINode::Create(BO.getType(), 2);
2025 NewPhi->addIncoming(NewBO, OtherBB);
2026 NewPhi->addIncoming(NewC, ConstBB);
2027 return NewPhi;
2028}
2029
2031 if (!isa<Constant>(I.getOperand(1)))
2032 return nullptr;
2033
2034 if (auto *Sel = dyn_cast<SelectInst>(I.getOperand(0))) {
2035 if (Instruction *NewSel = FoldOpIntoSelect(I, Sel))
2036 return NewSel;
2037 } else if (auto *PN = dyn_cast<PHINode>(I.getOperand(0))) {
2038 if (Instruction *NewPhi = foldOpIntoPhi(I, PN))
2039 return NewPhi;
2040 }
2041 return nullptr;
2042}
2043
2045 // If this GEP has only 0 indices, it is the same pointer as
2046 // Src. If Src is not a trivial GEP too, don't combine
2047 // the indices.
2048 if (GEP.hasAllZeroIndices() && !Src.hasAllZeroIndices() &&
2049 !Src.hasOneUse())
2050 return false;
2051 return true;
2052}
2053
2055 if (!isa<VectorType>(Inst.getType()))
2056 return nullptr;
2057
2058 BinaryOperator::BinaryOps Opcode = Inst.getOpcode();
2059 Value *LHS = Inst.getOperand(0), *RHS = Inst.getOperand(1);
2060 assert(cast<VectorType>(LHS->getType())->getElementCount() ==
2061 cast<VectorType>(Inst.getType())->getElementCount());
2062 assert(cast<VectorType>(RHS->getType())->getElementCount() ==
2063 cast<VectorType>(Inst.getType())->getElementCount());
2064
2065 // If both operands of the binop are vector concatenations, then perform the
2066 // narrow binop on each pair of the source operands followed by concatenation
2067 // of the results.
2068 Value *L0, *L1, *R0, *R1;
2069 ArrayRef<int> Mask;
2070 if (match(LHS, m_Shuffle(m_Value(L0), m_Value(L1), m_Mask(Mask))) &&
2071 match(RHS, m_Shuffle(m_Value(R0), m_Value(R1), m_SpecificMask(Mask))) &&
2072 LHS->hasOneUse() && RHS->hasOneUse() &&
2073 cast<ShuffleVectorInst>(LHS)->isConcat() &&
2074 cast<ShuffleVectorInst>(RHS)->isConcat()) {
2075 // This transform does not have the speculative execution constraint as
2076 // below because the shuffle is a concatenation. The new binops are
2077 // operating on exactly the same elements as the existing binop.
2078 // TODO: We could ease the mask requirement to allow different undef lanes,
2079 // but that requires an analysis of the binop-with-undef output value.
2080 Value *NewBO0 = Builder.CreateBinOp(Opcode, L0, R0);
2081 if (auto *BO = dyn_cast<BinaryOperator>(NewBO0))
2082 BO->copyIRFlags(&Inst);
2083 Value *NewBO1 = Builder.CreateBinOp(Opcode, L1, R1);
2084 if (auto *BO = dyn_cast<BinaryOperator>(NewBO1))
2085 BO->copyIRFlags(&Inst);
2086 return new ShuffleVectorInst(NewBO0, NewBO1, Mask);
2087 }
2088
2089 auto createBinOpReverse = [&](Value *X, Value *Y) {
2090 Value *V = Builder.CreateBinOp(Opcode, X, Y, Inst.getName());
2091 if (auto *BO = dyn_cast<BinaryOperator>(V))
2092 BO->copyIRFlags(&Inst);
2093 Module *M = Inst.getModule();
2095 M, Intrinsic::vector_reverse, V->getType());
2096 return CallInst::Create(F, V);
2097 };
2098
2099 // NOTE: Reverse shuffles don't require the speculative execution protection
2100 // below because they don't affect which lanes take part in the computation.
2101
2102 Value *V1, *V2;
2103 if (match(LHS, m_VecReverse(m_Value(V1)))) {
2104 // Op(rev(V1), rev(V2)) -> rev(Op(V1, V2))
2105 if (match(RHS, m_VecReverse(m_Value(V2))) &&
2106 (LHS->hasOneUse() || RHS->hasOneUse() ||
2107 (LHS == RHS && LHS->hasNUses(2))))
2108 return createBinOpReverse(V1, V2);
2109
2110 // Op(rev(V1), RHSSplat)) -> rev(Op(V1, RHSSplat))
2111 if (LHS->hasOneUse() && isSplatValue(RHS))
2112 return createBinOpReverse(V1, RHS);
2113 }
2114 // Op(LHSSplat, rev(V2)) -> rev(Op(LHSSplat, V2))
2115 else if (isSplatValue(LHS) && match(RHS, m_OneUse(m_VecReverse(m_Value(V2)))))
2116 return createBinOpReverse(LHS, V2);
2117
2118 // It may not be safe to reorder shuffles and things like div, urem, etc.
2119 // because we may trap when executing those ops on unknown vector elements.
2120 // See PR20059.
2122 return nullptr;
2123
2124 auto createBinOpShuffle = [&](Value *X, Value *Y, ArrayRef<int> M) {
2125 Value *XY = Builder.CreateBinOp(Opcode, X, Y);
2126 if (auto *BO = dyn_cast<BinaryOperator>(XY))
2127 BO->copyIRFlags(&Inst);
2128 return new ShuffleVectorInst(XY, M);
2129 };
2130
2131 // If both arguments of the binary operation are shuffles that use the same
2132 // mask and shuffle within a single vector, move the shuffle after the binop.
2133 if (match(LHS, m_Shuffle(m_Value(V1), m_Poison(), m_Mask(Mask))) &&
2134 match(RHS, m_Shuffle(m_Value(V2), m_Poison(), m_SpecificMask(Mask))) &&
2135 V1->getType() == V2->getType() &&
2136 (LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) {
2137 // Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask)
2138 return createBinOpShuffle(V1, V2, Mask);
2139 }
2140
2141 // If both arguments of a commutative binop are select-shuffles that use the
2142 // same mask with commuted operands, the shuffles are unnecessary.
2143 if (Inst.isCommutative() &&
2144 match(LHS, m_Shuffle(m_Value(V1), m_Value(V2), m_Mask(Mask))) &&
2145 match(RHS,
2146 m_Shuffle(m_Specific(V2), m_Specific(V1), m_SpecificMask(Mask)))) {
2147 auto *LShuf = cast<ShuffleVectorInst>(LHS);
2148 auto *RShuf = cast<ShuffleVectorInst>(RHS);
2149 // TODO: Allow shuffles that contain undefs in the mask?
2150 // That is legal, but it reduces undef knowledge.
2151 // TODO: Allow arbitrary shuffles by shuffling after binop?
2152 // That might be legal, but we have to deal with poison.
2153 if (LShuf->isSelect() &&
2154 !is_contained(LShuf->getShuffleMask(), PoisonMaskElem) &&
2155 RShuf->isSelect() &&
2156 !is_contained(RShuf->getShuffleMask(), PoisonMaskElem)) {
2157 // Example:
2158 // LHS = shuffle V1, V2, <0, 5, 6, 3>
2159 // RHS = shuffle V2, V1, <0, 5, 6, 3>
2160 // LHS + RHS --> (V10+V20, V21+V11, V22+V12, V13+V23) --> V1 + V2
2161 Instruction *NewBO = BinaryOperator::Create(Opcode, V1, V2);
2162 NewBO->copyIRFlags(&Inst);
2163 return NewBO;
2164 }
2165 }
2166
2167 // If one argument is a shuffle within one vector and the other is a constant,
2168 // try moving the shuffle after the binary operation. This canonicalization
2169 // intends to move shuffles closer to other shuffles and binops closer to
2170 // other binops, so they can be folded. It may also enable demanded elements
2171 // transforms.
2172 Constant *C;
2173 auto *InstVTy = dyn_cast<FixedVectorType>(Inst.getType());
2174 if (InstVTy &&
2176 m_Mask(Mask))),
2177 m_ImmConstant(C))) &&
2178 cast<FixedVectorType>(V1->getType())->getNumElements() <=
2179 InstVTy->getNumElements()) {
2180 assert(InstVTy->getScalarType() == V1->getType()->getScalarType() &&
2181 "Shuffle should not change scalar type");
2182
2183 // Find constant NewC that has property:
2184 // shuffle(NewC, ShMask) = C
2185 // If such constant does not exist (example: ShMask=<0,0> and C=<1,2>)
2186 // reorder is not possible. A 1-to-1 mapping is not required. Example:
2187 // ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = <undef,5,6,undef>
2188 bool ConstOp1 = isa<Constant>(RHS);
2189 ArrayRef<int> ShMask = Mask;
2190 unsigned SrcVecNumElts =
2191 cast<FixedVectorType>(V1->getType())->getNumElements();
2192 PoisonValue *PoisonScalar = PoisonValue::get(C->getType()->getScalarType());
2193 SmallVector<Constant *, 16> NewVecC(SrcVecNumElts, PoisonScalar);
2194 bool MayChange = true;
2195 unsigned NumElts = InstVTy->getNumElements();
2196 for (unsigned I = 0; I < NumElts; ++I) {
2197 Constant *CElt = C->getAggregateElement(I);
2198 if (ShMask[I] >= 0) {
2199 assert(ShMask[I] < (int)NumElts && "Not expecting narrowing shuffle");
2200 Constant *NewCElt = NewVecC[ShMask[I]];
2201 // Bail out if:
2202 // 1. The constant vector contains a constant expression.
2203 // 2. The shuffle needs an element of the constant vector that can't
2204 // be mapped to a new constant vector.
2205 // 3. This is a widening shuffle that copies elements of V1 into the
2206 // extended elements (extending with poison is allowed).
2207 if (!CElt || (!isa<PoisonValue>(NewCElt) && NewCElt != CElt) ||
2208 I >= SrcVecNumElts) {
2209 MayChange = false;
2210 break;
2211 }
2212 NewVecC[ShMask[I]] = CElt;
2213 }
2214 // If this is a widening shuffle, we must be able to extend with poison
2215 // elements. If the original binop does not produce a poison in the high
2216 // lanes, then this transform is not safe.
2217 // Similarly for poison lanes due to the shuffle mask, we can only
2218 // transform binops that preserve poison.
2219 // TODO: We could shuffle those non-poison constant values into the
2220 // result by using a constant vector (rather than an poison vector)
2221 // as operand 1 of the new binop, but that might be too aggressive
2222 // for target-independent shuffle creation.
2223 if (I >= SrcVecNumElts || ShMask[I] < 0) {
2224 Constant *MaybePoison =
2225 ConstOp1
2226 ? ConstantFoldBinaryOpOperands(Opcode, PoisonScalar, CElt, DL)
2227 : ConstantFoldBinaryOpOperands(Opcode, CElt, PoisonScalar, DL);
2228 if (!MaybePoison || !isa<PoisonValue>(MaybePoison)) {
2229 MayChange = false;
2230 break;
2231 }
2232 }
2233 }
2234 if (MayChange) {
2235 Constant *NewC = ConstantVector::get(NewVecC);
2236 // It may not be safe to execute a binop on a vector with poison elements
2237 // because the entire instruction can be folded to undef or create poison
2238 // that did not exist in the original code.
2239 // TODO: The shift case should not be necessary.
2240 if (Inst.isIntDivRem() || (Inst.isShift() && ConstOp1))
2241 NewC = getSafeVectorConstantForBinop(Opcode, NewC, ConstOp1);
2242
2243 // Op(shuffle(V1, Mask), C) -> shuffle(Op(V1, NewC), Mask)
2244 // Op(C, shuffle(V1, Mask)) -> shuffle(Op(NewC, V1), Mask)
2245 Value *NewLHS = ConstOp1 ? V1 : NewC;
2246 Value *NewRHS = ConstOp1 ? NewC : V1;
2247 return createBinOpShuffle(NewLHS, NewRHS, Mask);
2248 }
2249 }
2250
2251 // Try to reassociate to sink a splat shuffle after a binary operation.
2252 if (Inst.isAssociative() && Inst.isCommutative()) {
2253 // Canonicalize shuffle operand as LHS.
2254 if (isa<ShuffleVectorInst>(RHS))
2255 std::swap(LHS, RHS);
2256
2257 Value *X;
2258 ArrayRef<int> MaskC;
2259 int SplatIndex;
2260 Value *Y, *OtherOp;
2261 if (!match(LHS,
2262 m_OneUse(m_Shuffle(m_Value(X), m_Undef(), m_Mask(MaskC)))) ||
2263 !match(MaskC, m_SplatOrPoisonMask(SplatIndex)) ||
2264 X->getType() != Inst.getType() ||
2265 !match(RHS, m_OneUse(m_BinOp(Opcode, m_Value(Y), m_Value(OtherOp)))))
2266 return nullptr;
2267
2268 // FIXME: This may not be safe if the analysis allows undef elements. By
2269 // moving 'Y' before the splat shuffle, we are implicitly assuming
2270 // that it is not undef/poison at the splat index.
2271 if (isSplatValue(OtherOp, SplatIndex)) {
2272 std::swap(Y, OtherOp);
2273 } else if (!isSplatValue(Y, SplatIndex)) {
2274 return nullptr;
2275 }
2276
2277 // X and Y are splatted values, so perform the binary operation on those
2278 // values followed by a splat followed by the 2nd binary operation:
2279 // bo (splat X), (bo Y, OtherOp) --> bo (splat (bo X, Y)), OtherOp
2280 Value *NewBO = Builder.CreateBinOp(Opcode, X, Y);
2281 SmallVector<int, 8> NewMask(MaskC.size(), SplatIndex);
2282 Value *NewSplat = Builder.CreateShuffleVector(NewBO, NewMask);
2283 Instruction *R = BinaryOperator::Create(Opcode, NewSplat, OtherOp);
2284
2285 // Intersect FMF on both new binops. Other (poison-generating) flags are
2286 // dropped to be safe.
2287 if (isa<FPMathOperator>(R)) {
2288 R->copyFastMathFlags(&Inst);
2289 R->andIRFlags(RHS);
2290 }
2291 if (auto *NewInstBO = dyn_cast<BinaryOperator>(NewBO))
2292 NewInstBO->copyIRFlags(R);
2293 return R;
2294 }
2295
2296 return nullptr;
2297}
2298
2299/// Try to narrow the width of a binop if at least 1 operand is an extend of
2300/// of a value. This requires a potentially expensive known bits check to make
2301/// sure the narrow op does not overflow.
2302Instruction *InstCombinerImpl::narrowMathIfNoOverflow(BinaryOperator &BO) {
2303 // We need at least one extended operand.
2304 Value *Op0 = BO.getOperand(0), *Op1 = BO.getOperand(1);
2305
2306 // If this is a sub, we swap the operands since we always want an extension
2307 // on the RHS. The LHS can be an extension or a constant.
2308 if (BO.getOpcode() == Instruction::Sub)
2309 std::swap(Op0, Op1);
2310
2311 Value *X;
2312 bool IsSext = match(Op0, m_SExt(m_Value(X)));
2313 if (!IsSext && !match(Op0, m_ZExt(m_Value(X))))
2314 return nullptr;
2315
2316 // If both operands are the same extension from the same source type and we
2317 // can eliminate at least one (hasOneUse), this might work.
2318 CastInst::CastOps CastOpc = IsSext ? Instruction::SExt : Instruction::ZExt;
2319 Value *Y;
2320 if (!(match(Op1, m_ZExtOrSExt(m_Value(Y))) && X->getType() == Y->getType() &&
2321 cast<Operator>(Op1)->getOpcode() == CastOpc &&
2322 (Op0->hasOneUse() || Op1->hasOneUse()))) {
2323 // If that did not match, see if we have a suitable constant operand.
2324 // Truncating and extending must produce the same constant.
2325 Constant *WideC;
2326 if (!Op0->hasOneUse() || !match(Op1, m_Constant(WideC)))
2327 return nullptr;
2328 Constant *NarrowC = getLosslessTrunc(WideC, X->getType(), CastOpc);
2329 if (!NarrowC)
2330 return nullptr;
2331 Y = NarrowC;
2332 }
2333
2334 // Swap back now that we found our operands.
2335 if (BO.getOpcode() == Instruction::Sub)
2336 std::swap(X, Y);
2337
2338 // Both operands have narrow versions. Last step: the math must not overflow
2339 // in the narrow width.
2340 if (!willNotOverflow(BO.getOpcode(), X, Y, BO, IsSext))
2341 return nullptr;
2342
2343 // bo (ext X), (ext Y) --> ext (bo X, Y)
2344 // bo (ext X), C --> ext (bo X, C')
2345 Value *NarrowBO = Builder.CreateBinOp(BO.getOpcode(), X, Y, "narrow");
2346 if (auto *NewBinOp = dyn_cast<BinaryOperator>(NarrowBO)) {
2347 if (IsSext)
2348 NewBinOp->setHasNoSignedWrap();
2349 else
2350 NewBinOp->setHasNoUnsignedWrap();
2351 }
2352 return CastInst::Create(CastOpc, NarrowBO, BO.getType());
2353}
2354
2355/// Determine nowrap flags for (gep (gep p, x), y) to (gep p, (x + y))
2356/// transform.
2358 GEPOperator &GEP2) {
2360}
2361
2362/// Thread a GEP operation with constant indices through the constant true/false
2363/// arms of a select.
2365 InstCombiner::BuilderTy &Builder) {
2366 if (!GEP.hasAllConstantIndices())
2367 return nullptr;
2368
2369 Instruction *Sel;
2370 Value *Cond;
2371 Constant *TrueC, *FalseC;
2372 if (!match(GEP.getPointerOperand(), m_Instruction(Sel)) ||
2373 !match(Sel,
2374 m_Select(m_Value(Cond), m_Constant(TrueC), m_Constant(FalseC))))
2375 return nullptr;
2376
2377 // gep (select Cond, TrueC, FalseC), IndexC --> select Cond, TrueC', FalseC'
2378 // Propagate 'inbounds' and metadata from existing instructions.
2379 // Note: using IRBuilder to create the constants for efficiency.
2380 SmallVector<Value *, 4> IndexC(GEP.indices());
2381 GEPNoWrapFlags NW = GEP.getNoWrapFlags();
2382 Type *Ty = GEP.getSourceElementType();
2383 Value *NewTrueC = Builder.CreateGEP(Ty, TrueC, IndexC, "", NW);
2384 Value *NewFalseC = Builder.CreateGEP(Ty, FalseC, IndexC, "", NW);
2385 return SelectInst::Create(Cond, NewTrueC, NewFalseC, "", nullptr, Sel);
2386}
2387
2388// Canonicalization:
2389// gep T, (gep i8, base, C1), (Index + C2) into
2390// gep T, (gep i8, base, C1 + C2 * sizeof(T)), Index
2392 GEPOperator *Src,
2393 InstCombinerImpl &IC) {
2394 if (GEP.getNumIndices() != 1)
2395 return nullptr;
2396 auto &DL = IC.getDataLayout();
2397 Value *Base;
2398 const APInt *C1;
2399 if (!match(Src, m_PtrAdd(m_Value(Base), m_APInt(C1))))
2400 return nullptr;
2401 Value *VarIndex;
2402 const APInt *C2;
2403 Type *PtrTy = Src->getType()->getScalarType();
2404 unsigned IndexSizeInBits = DL.getIndexTypeSizeInBits(PtrTy);
2405 if (!match(GEP.getOperand(1), m_AddLike(m_Value(VarIndex), m_APInt(C2))))
2406 return nullptr;
2407 if (C1->getBitWidth() != IndexSizeInBits ||
2408 C2->getBitWidth() != IndexSizeInBits)
2409 return nullptr;
2410 Type *BaseType = GEP.getSourceElementType();
2411 if (isa<ScalableVectorType>(BaseType))
2412 return nullptr;
2413 APInt TypeSize(IndexSizeInBits, DL.getTypeAllocSize(BaseType));
2414 APInt NewOffset = TypeSize * *C2 + *C1;
2415 if (NewOffset.isZero() ||
2416 (Src->hasOneUse() && GEP.getOperand(1)->hasOneUse())) {
2417 Value *GEPConst =
2418 IC.Builder.CreatePtrAdd(Base, IC.Builder.getInt(NewOffset));
2419 return GetElementPtrInst::Create(BaseType, GEPConst, VarIndex);
2420 }
2421
2422 return nullptr;
2423}
2424
2426 GEPOperator *Src) {
2427 // Combine Indices - If the source pointer to this getelementptr instruction
2428 // is a getelementptr instruction with matching element type, combine the
2429 // indices of the two getelementptr instructions into a single instruction.
2430 if (!shouldMergeGEPs(*cast<GEPOperator>(&GEP), *Src))
2431 return nullptr;
2432
2433 if (auto *I = canonicalizeGEPOfConstGEPI8(GEP, Src, *this))
2434 return I;
2435
2436 // For constant GEPs, use a more general offset-based folding approach.
2437 Type *PtrTy = Src->getType()->getScalarType();
2438 if (GEP.hasAllConstantIndices() &&
2439 (Src->hasOneUse() || Src->hasAllConstantIndices())) {
2440 // Split Src into a variable part and a constant suffix.
2442 Type *BaseType = GTI.getIndexedType();
2443 bool IsFirstType = true;
2444 unsigned NumVarIndices = 0;
2445 for (auto Pair : enumerate(Src->indices())) {
2446 if (!isa<ConstantInt>(Pair.value())) {
2447 BaseType = GTI.getIndexedType();
2448 IsFirstType = false;
2449 NumVarIndices = Pair.index() + 1;
2450 }
2451 ++GTI;
2452 }
2453
2454 // Determine the offset for the constant suffix of Src.
2456 if (NumVarIndices != Src->getNumIndices()) {
2457 // FIXME: getIndexedOffsetInType() does not handled scalable vectors.
2458 if (BaseType->isScalableTy())
2459 return nullptr;
2460
2461 SmallVector<Value *> ConstantIndices;
2462 if (!IsFirstType)
2463 ConstantIndices.push_back(
2465 append_range(ConstantIndices, drop_begin(Src->indices(), NumVarIndices));
2466 Offset += DL.getIndexedOffsetInType(BaseType, ConstantIndices);
2467 }
2468
2469 // Add the offset for GEP (which is fully constant).
2470 if (!GEP.accumulateConstantOffset(DL, Offset))
2471 return nullptr;
2472
2473 // Convert the total offset back into indices.
2474 SmallVector<APInt> ConstIndices =
2476 if (!Offset.isZero() || (!IsFirstType && !ConstIndices[0].isZero()))
2477 return nullptr;
2478
2479 GEPNoWrapFlags NW = getMergedGEPNoWrapFlags(*Src, *cast<GEPOperator>(&GEP));
2480 SmallVector<Value *> Indices;
2481 append_range(Indices, drop_end(Src->indices(),
2482 Src->getNumIndices() - NumVarIndices));
2483 for (const APInt &Idx : drop_begin(ConstIndices, !IsFirstType)) {
2484 Indices.push_back(ConstantInt::get(GEP.getContext(), Idx));
2485 // Even if the total offset is inbounds, we may end up representing it
2486 // by first performing a larger negative offset, and then a smaller
2487 // positive one. The large negative offset might go out of bounds. Only
2488 // preserve inbounds if all signs are the same.
2489 if (Idx.isNonNegative() != ConstIndices[0].isNonNegative())
2491 if (!Idx.isNonNegative())
2492 NW = NW.withoutNoUnsignedWrap();
2493 }
2494
2495 return replaceInstUsesWith(
2496 GEP, Builder.CreateGEP(Src->getSourceElementType(), Src->getOperand(0),
2497 Indices, "", NW));
2498 }
2499
2500 if (Src->getResultElementType() != GEP.getSourceElementType())
2501 return nullptr;
2502
2503 SmallVector<Value*, 8> Indices;
2504
2505 // Find out whether the last index in the source GEP is a sequential idx.
2506 bool EndsWithSequential = false;
2507 for (gep_type_iterator I = gep_type_begin(*Src), E = gep_type_end(*Src);
2508 I != E; ++I)
2509 EndsWithSequential = I.isSequential();
2510
2511 // Can we combine the two pointer arithmetics offsets?
2512 if (EndsWithSequential) {
2513 // Replace: gep (gep %P, long B), long A, ...
2514 // With: T = long A+B; gep %P, T, ...
2515 Value *SO1 = Src->getOperand(Src->getNumOperands()-1);
2516 Value *GO1 = GEP.getOperand(1);
2517
2518 // If they aren't the same type, then the input hasn't been processed
2519 // by the loop above yet (which canonicalizes sequential index types to
2520 // intptr_t). Just avoid transforming this until the input has been
2521 // normalized.
2522 if (SO1->getType() != GO1->getType())
2523 return nullptr;
2524
2525 Value *Sum =
2526 simplifyAddInst(GO1, SO1, false, false, SQ.getWithInstruction(&GEP));
2527 // Only do the combine when we are sure the cost after the
2528 // merge is never more than that before the merge.
2529 if (Sum == nullptr)
2530 return nullptr;
2531
2532 Indices.append(Src->op_begin()+1, Src->op_end()-1);
2533 Indices.push_back(Sum);
2534 Indices.append(GEP.op_begin()+2, GEP.op_end());
2535 } else if (isa<Constant>(*GEP.idx_begin()) &&
2536 cast<Constant>(*GEP.idx_begin())->isNullValue() &&
2537 Src->getNumOperands() != 1) {
2538 // Otherwise we can do the fold if the first index of the GEP is a zero
2539 Indices.append(Src->op_begin()+1, Src->op_end());
2540 Indices.append(GEP.idx_begin()+1, GEP.idx_end());
2541 }
2542
2543 if (!Indices.empty())
2544 return replaceInstUsesWith(
2546 Src->getSourceElementType(), Src->getOperand(0), Indices, "",
2547 getMergedGEPNoWrapFlags(*Src, *cast<GEPOperator>(&GEP))));
2548
2549 return nullptr;
2550}
2551
2553 BuilderTy *Builder,
2554 bool &DoesConsume, unsigned Depth) {
2555 static Value *const NonNull = reinterpret_cast<Value *>(uintptr_t(1));
2556 // ~(~(X)) -> X.
2557 Value *A, *B;
2558 if (match(V, m_Not(m_Value(A)))) {
2559 DoesConsume = true;
2560 return A;
2561 }
2562
2563 Constant *C;
2564 // Constants can be considered to be not'ed values.
2565 if (match(V, m_ImmConstant(C)))
2566 return ConstantExpr::getNot(C);
2567
2569 return nullptr;
2570
2571 // The rest of the cases require that we invert all uses so don't bother
2572 // doing the analysis if we know we can't use the result.
2573 if (!WillInvertAllUses)
2574 return nullptr;
2575
2576 // Compares can be inverted if all of their uses are being modified to use
2577 // the ~V.
2578 if (auto *I = dyn_cast<CmpInst>(V)) {
2579 if (Builder != nullptr)
2580 return Builder->CreateCmp(I->getInversePredicate(), I->getOperand(0),
2581 I->getOperand(1));
2582 return NonNull;
2583 }
2584
2585 // If `V` is of the form `A + B` then `-1 - V` can be folded into
2586 // `(-1 - B) - A` if we are willing to invert all of the uses.
2587 if (match(V, m_Add(m_Value(A), m_Value(B)))) {
2588 if (auto *BV = getFreelyInvertedImpl(B, B->hasOneUse(), Builder,
2589 DoesConsume, Depth))
2590 return Builder ? Builder->CreateSub(BV, A) : NonNull;
2591 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2592 DoesConsume, Depth))
2593 return Builder ? Builder->CreateSub(AV, B) : NonNull;
2594 return nullptr;
2595 }
2596
2597 // If `V` is of the form `A ^ ~B` then `~(A ^ ~B)` can be folded
2598 // into `A ^ B` if we are willing to invert all of the uses.
2599 if (match(V, m_Xor(m_Value(A), m_Value(B)))) {
2600 if (auto *BV = getFreelyInvertedImpl(B, B->hasOneUse(), Builder,
2601 DoesConsume, Depth))
2602 return Builder ? Builder->CreateXor(A, BV) : NonNull;
2603 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2604 DoesConsume, Depth))
2605 return Builder ? Builder->CreateXor(AV, B) : NonNull;
2606 return nullptr;
2607 }
2608
2609 // If `V` is of the form `B - A` then `-1 - V` can be folded into
2610 // `A + (-1 - B)` if we are willing to invert all of the uses.
2611 if (match(V, m_Sub(m_Value(A), m_Value(B)))) {
2612 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2613 DoesConsume, Depth))
2614 return Builder ? Builder->CreateAdd(AV, B) : NonNull;
2615 return nullptr;
2616 }
2617
2618 // If `V` is of the form `(~A) s>> B` then `~((~A) s>> B)` can be folded
2619 // into `A s>> B` if we are willing to invert all of the uses.
2620 if (match(V, m_AShr(m_Value(A), m_Value(B)))) {
2621 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2622 DoesConsume, Depth))
2623 return Builder ? Builder->CreateAShr(AV, B) : NonNull;
2624 return nullptr;
2625 }
2626
2627 Value *Cond;
2628 // LogicOps are special in that we canonicalize them at the cost of an
2629 // instruction.
2630 bool IsSelect = match(V, m_Select(m_Value(Cond), m_Value(A), m_Value(B))) &&
2631 !shouldAvoidAbsorbingNotIntoSelect(*cast<SelectInst>(V));
2632 // Selects/min/max with invertible operands are freely invertible
2633 if (IsSelect || match(V, m_MaxOrMin(m_Value(A), m_Value(B)))) {
2634 bool LocalDoesConsume = DoesConsume;
2635 if (!getFreelyInvertedImpl(B, B->hasOneUse(), /*Builder*/ nullptr,
2636 LocalDoesConsume, Depth))
2637 return nullptr;
2638 if (Value *NotA = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2639 LocalDoesConsume, Depth)) {
2640 DoesConsume = LocalDoesConsume;
2641 if (Builder != nullptr) {
2642 Value *NotB = getFreelyInvertedImpl(B, B->hasOneUse(), Builder,
2643 DoesConsume, Depth);
2644 assert(NotB != nullptr &&
2645 "Unable to build inverted value for known freely invertable op");
2646 if (auto *II = dyn_cast<IntrinsicInst>(V))
2648 getInverseMinMaxIntrinsic(II->getIntrinsicID()), NotA, NotB);
2649 return Builder->CreateSelect(Cond, NotA, NotB);
2650 }
2651 return NonNull;
2652 }
2653 }
2654
2655 if (PHINode *PN = dyn_cast<PHINode>(V)) {
2656 bool LocalDoesConsume = DoesConsume;
2658 for (Use &U : PN->operands()) {
2659 BasicBlock *IncomingBlock = PN->getIncomingBlock(U);
2660 Value *NewIncomingVal = getFreelyInvertedImpl(
2661 U.get(), /*WillInvertAllUses=*/false,
2662 /*Builder=*/nullptr, LocalDoesConsume, MaxAnalysisRecursionDepth - 1);
2663 if (NewIncomingVal == nullptr)
2664 return nullptr;
2665 // Make sure that we can safely erase the original PHI node.
2666 if (NewIncomingVal == V)
2667 return nullptr;
2668 if (Builder != nullptr)
2669 IncomingValues.emplace_back(NewIncomingVal, IncomingBlock);
2670 }
2671
2672 DoesConsume = LocalDoesConsume;
2673 if (Builder != nullptr) {
2676 PHINode *NewPN =
2677 Builder->CreatePHI(PN->getType(), PN->getNumIncomingValues());
2678 for (auto [Val, Pred] : IncomingValues)
2679 NewPN->addIncoming(Val, Pred);
2680 return NewPN;
2681 }
2682 return NonNull;
2683 }
2684
2685 if (match(V, m_SExtLike(m_Value(A)))) {
2686 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2687 DoesConsume, Depth))
2688 return Builder ? Builder->CreateSExt(AV, V->getType()) : NonNull;
2689 return nullptr;
2690 }
2691
2692 if (match(V, m_Trunc(m_Value(A)))) {
2693 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2694 DoesConsume, Depth))
2695 return Builder ? Builder->CreateTrunc(AV, V->getType()) : NonNull;
2696 return nullptr;
2697 }
2698
2699 // De Morgan's Laws:
2700 // (~(A | B)) -> (~A & ~B)
2701 // (~(A & B)) -> (~A | ~B)
2702 auto TryInvertAndOrUsingDeMorgan = [&](Instruction::BinaryOps Opcode,
2703 bool IsLogical, Value *A,
2704 Value *B) -> Value * {
2705 bool LocalDoesConsume = DoesConsume;
2706 if (!getFreelyInvertedImpl(B, B->hasOneUse(), /*Builder=*/nullptr,
2707 LocalDoesConsume, Depth))
2708 return nullptr;
2709 if (auto *NotA = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2710 LocalDoesConsume, Depth)) {
2711 auto *NotB = getFreelyInvertedImpl(B, B->hasOneUse(), Builder,
2712 LocalDoesConsume, Depth);
2713 DoesConsume = LocalDoesConsume;
2714 if (IsLogical)
2715 return Builder ? Builder->CreateLogicalOp(Opcode, NotA, NotB) : NonNull;
2716 return Builder ? Builder->CreateBinOp(Opcode, NotA, NotB) : NonNull;
2717 }
2718
2719 return nullptr;
2720 };
2721
2722 if (match(V, m_Or(m_Value(A), m_Value(B))))
2723 return TryInvertAndOrUsingDeMorgan(Instruction::And, /*IsLogical=*/false, A,
2724 B);
2725
2726 if (match(V, m_And(m_Value(A), m_Value(B))))
2727 return TryInvertAndOrUsingDeMorgan(Instruction::Or, /*IsLogical=*/false, A,
2728 B);
2729
2730 if (match(V, m_LogicalOr(m_Value(A), m_Value(B))))
2731 return TryInvertAndOrUsingDeMorgan(Instruction::And, /*IsLogical=*/true, A,
2732 B);
2733
2734 if (match(V, m_LogicalAnd(m_Value(A), m_Value(B))))
2735 return TryInvertAndOrUsingDeMorgan(Instruction::Or, /*IsLogical=*/true, A,
2736 B);
2737
2738 return nullptr;
2739}
2740
2741/// Return true if we should canonicalize the gep to an i8 ptradd.
2743 Value *PtrOp = GEP.getOperand(0);
2744 Type *GEPEltType = GEP.getSourceElementType();
2745 if (GEPEltType->isIntegerTy(8))
2746 return false;
2747
2748 // Canonicalize scalable GEPs to an explicit offset using the llvm.vscale
2749 // intrinsic. This has better support in BasicAA.
2750 if (GEPEltType->isScalableTy())
2751 return true;
2752
2753 // gep i32 p, mul(O, C) -> gep i8, p, mul(O, C*4) to fold the two multiplies
2754 // together.
2755 if (GEP.getNumIndices() == 1 &&
2756 match(GEP.getOperand(1),
2758 m_Shl(m_Value(), m_ConstantInt())))))
2759 return true;
2760
2761 // gep (gep %p, C1), %x, C2 is expanded so the two constants can
2762 // possibly be merged together.
2763 auto PtrOpGep = dyn_cast<GEPOperator>(PtrOp);
2764 return PtrOpGep && PtrOpGep->hasAllConstantIndices() &&
2765 any_of(GEP.indices(), [](Value *V) {
2766 const APInt *C;
2767 return match(V, m_APInt(C)) && !C->isZero();
2768 });
2769}
2770
2772 IRBuilderBase &Builder) {
2773 auto *Op1 = dyn_cast<GetElementPtrInst>(PN->getOperand(0));
2774 if (!Op1)
2775 return nullptr;
2776
2777 // Don't fold a GEP into itself through a PHI node. This can only happen
2778 // through the back-edge of a loop. Folding a GEP into itself means that
2779 // the value of the previous iteration needs to be stored in the meantime,
2780 // thus requiring an additional register variable to be live, but not
2781 // actually achieving anything (the GEP still needs to be executed once per
2782 // loop iteration).
2783 if (Op1 == &GEP)
2784 return nullptr;
2785 GEPNoWrapFlags NW = Op1->getNoWrapFlags();
2786
2787 int DI = -1;
2788
2789 for (auto I = PN->op_begin()+1, E = PN->op_end(); I !=E; ++I) {
2790 auto *Op2 = dyn_cast<GetElementPtrInst>(*I);
2791 if (!Op2 || Op1->getNumOperands() != Op2->getNumOperands() ||
2792 Op1->getSourceElementType() != Op2->getSourceElementType())
2793 return nullptr;
2794
2795 // As for Op1 above, don't try to fold a GEP into itself.
2796 if (Op2 == &GEP)
2797 return nullptr;
2798
2799 // Keep track of the type as we walk the GEP.
2800 Type *CurTy = nullptr;
2801
2802 for (unsigned J = 0, F = Op1->getNumOperands(); J != F; ++J) {
2803 if (Op1->getOperand(J)->getType() != Op2->getOperand(J)->getType())
2804 return nullptr;
2805
2806 if (Op1->getOperand(J) != Op2->getOperand(J)) {
2807 if (DI == -1) {
2808 // We have not seen any differences yet in the GEPs feeding the
2809 // PHI yet, so we record this one if it is allowed to be a
2810 // variable.
2811
2812 // The first two arguments can vary for any GEP, the rest have to be
2813 // static for struct slots
2814 if (J > 1) {
2815 assert(CurTy && "No current type?");
2816 if (CurTy->isStructTy())
2817 return nullptr;
2818 }
2819
2820 DI = J;
2821 } else {
2822 // The GEP is different by more than one input. While this could be
2823 // extended to support GEPs that vary by more than one variable it
2824 // doesn't make sense since it greatly increases the complexity and
2825 // would result in an R+R+R addressing mode which no backend
2826 // directly supports and would need to be broken into several
2827 // simpler instructions anyway.
2828 return nullptr;
2829 }
2830 }
2831
2832 // Sink down a layer of the type for the next iteration.
2833 if (J > 0) {
2834 if (J == 1) {
2835 CurTy = Op1->getSourceElementType();
2836 } else {
2837 CurTy =
2838 GetElementPtrInst::getTypeAtIndex(CurTy, Op1->getOperand(J));
2839 }
2840 }
2841 }
2842
2843 NW &= Op2->getNoWrapFlags();
2844 }
2845
2846 // If not all GEPs are identical we'll have to create a new PHI node.
2847 // Check that the old PHI node has only one use so that it will get
2848 // removed.
2849 if (DI != -1 && !PN->hasOneUse())
2850 return nullptr;
2851
2852 auto *NewGEP = cast<GetElementPtrInst>(Op1->clone());
2853 NewGEP->setNoWrapFlags(NW);
2854
2855 if (DI == -1) {
2856 // All the GEPs feeding the PHI are identical. Clone one down into our
2857 // BB so that it can be merged with the current GEP.
2858 } else {
2859 // All the GEPs feeding the PHI differ at a single offset. Clone a GEP
2860 // into the current block so it can be merged, and create a new PHI to
2861 // set that index.
2862 PHINode *NewPN;
2863 {
2864 IRBuilderBase::InsertPointGuard Guard(Builder);
2865 Builder.SetInsertPoint(PN);
2866 NewPN = Builder.CreatePHI(Op1->getOperand(DI)->getType(),
2867 PN->getNumOperands());
2868 }
2869
2870 for (auto &I : PN->operands())
2871 NewPN->addIncoming(cast<GEPOperator>(I)->getOperand(DI),
2872 PN->getIncomingBlock(I));
2873
2874 NewGEP->setOperand(DI, NewPN);
2875 }
2876
2877 NewGEP->insertBefore(*GEP.getParent(), GEP.getParent()->getFirstInsertionPt());
2878 return NewGEP;
2879}
2880
2882 Value *PtrOp = GEP.getOperand(0);
2883 SmallVector<Value *, 8> Indices(GEP.indices());
2884 Type *GEPType = GEP.getType();
2885 Type *GEPEltType = GEP.getSourceElementType();
2886 if (Value *V =
2887 simplifyGEPInst(GEPEltType, PtrOp, Indices, GEP.getNoWrapFlags(),
2889 return replaceInstUsesWith(GEP, V);
2890
2891 // For vector geps, use the generic demanded vector support.
2892 // Skip if GEP return type is scalable. The number of elements is unknown at
2893 // compile-time.
2894 if (auto *GEPFVTy = dyn_cast<FixedVectorType>(GEPType)) {
2895 auto VWidth = GEPFVTy->getNumElements();
2896 APInt PoisonElts(VWidth, 0);
2897 APInt AllOnesEltMask(APInt::getAllOnes(VWidth));
2898 if (Value *V = SimplifyDemandedVectorElts(&GEP, AllOnesEltMask,
2899 PoisonElts)) {
2900 if (V != &GEP)
2901 return replaceInstUsesWith(GEP, V);
2902 return &GEP;
2903 }
2904
2905 // TODO: 1) Scalarize splat operands, 2) scalarize entire instruction if
2906 // possible (decide on canonical form for pointer broadcast), 3) exploit
2907 // undef elements to decrease demanded bits
2908 }
2909
2910 // Eliminate unneeded casts for indices, and replace indices which displace
2911 // by multiples of a zero size type with zero.
2912 bool MadeChange = false;
2913
2914 // Index width may not be the same width as pointer width.
2915 // Data layout chooses the right type based on supported integer types.
2916 Type *NewScalarIndexTy =
2917 DL.getIndexType(GEP.getPointerOperandType()->getScalarType());
2918
2920 for (User::op_iterator I = GEP.op_begin() + 1, E = GEP.op_end(); I != E;
2921 ++I, ++GTI) {
2922 // Skip indices into struct types.
2923 if (GTI.isStruct())
2924 continue;
2925
2926 Type *IndexTy = (*I)->getType();
2927 Type *NewIndexType =
2928 IndexTy->isVectorTy()
2929 ? VectorType::get(NewScalarIndexTy,
2930 cast<VectorType>(IndexTy)->getElementCount())
2931 : NewScalarIndexTy;
2932
2933 // If the element type has zero size then any index over it is equivalent
2934 // to an index of zero, so replace it with zero if it is not zero already.
2935 Type *EltTy = GTI.getIndexedType();
2936 if (EltTy->isSized() && DL.getTypeAllocSize(EltTy).isZero())
2937 if (!isa<Constant>(*I) || !match(I->get(), m_Zero())) {
2938 *I = Constant::getNullValue(NewIndexType);
2939 MadeChange = true;
2940 }
2941
2942 if (IndexTy != NewIndexType) {
2943 // If we are using a wider index than needed for this platform, shrink
2944 // it to what we need. If narrower, sign-extend it to what we need.
2945 // This explicit cast can make subsequent optimizations more obvious.
2946 *I = Builder.CreateIntCast(*I, NewIndexType, true);
2947 MadeChange = true;
2948 }
2949 }
2950 if (MadeChange)
2951 return &GEP;
2952
2953 // Canonicalize constant GEPs to i8 type.
2954 if (!GEPEltType->isIntegerTy(8) && GEP.hasAllConstantIndices()) {
2956 if (GEP.accumulateConstantOffset(DL, Offset))
2957 return replaceInstUsesWith(
2959 GEP.getNoWrapFlags()));
2960 }
2961
2963 Value *Offset = EmitGEPOffset(cast<GEPOperator>(&GEP));
2964 Value *NewGEP =
2965 Builder.CreatePtrAdd(PtrOp, Offset, "", GEP.getNoWrapFlags());
2966 return replaceInstUsesWith(GEP, NewGEP);
2967 }
2968
2969 // Check to see if the inputs to the PHI node are getelementptr instructions.
2970 if (auto *PN = dyn_cast<PHINode>(PtrOp)) {
2971 if (Value *NewPtrOp = foldGEPOfPhi(GEP, PN, Builder))
2972 return replaceOperand(GEP, 0, NewPtrOp);
2973 }
2974
2975 if (auto *Src = dyn_cast<GEPOperator>(PtrOp))
2976 if (Instruction *I = visitGEPOfGEP(GEP, Src))
2977 return I;
2978
2979 if (GEP.getNumIndices() == 1) {
2980 unsigned AS = GEP.getPointerAddressSpace();
2981 if (GEP.getOperand(1)->getType()->getScalarSizeInBits() ==
2982 DL.getIndexSizeInBits(AS)) {
2983 uint64_t TyAllocSize = DL.getTypeAllocSize(GEPEltType).getFixedValue();
2984
2985 if (TyAllocSize == 1) {
2986 // Canonicalize (gep i8* X, (ptrtoint Y)-(ptrtoint X)) to (bitcast Y),
2987 // but only if the result pointer is only used as if it were an integer,
2988 // or both point to the same underlying object (otherwise provenance is
2989 // not necessarily retained).
2990 Value *X = GEP.getPointerOperand();
2991 Value *Y;
2992 if (match(GEP.getOperand(1),
2994 GEPType == Y->getType()) {
2995 bool HasSameUnderlyingObject =
2997 bool Changed = false;
2998 GEP.replaceUsesWithIf(Y, [&](Use &U) {
2999 bool ShouldReplace = HasSameUnderlyingObject ||
3000 isa<ICmpInst>(U.getUser()) ||
3001 isa<PtrToIntInst>(U.getUser());
3002 Changed |= ShouldReplace;
3003 return ShouldReplace;
3004 });
3005 return Changed ? &GEP : nullptr;
3006 }
3007 } else if (auto *ExactIns =
3008 dyn_cast<PossiblyExactOperator>(GEP.getOperand(1))) {
3009 // Canonicalize (gep T* X, V / sizeof(T)) to (gep i8* X, V)
3010 Value *V;
3011 if (ExactIns->isExact()) {
3012 if ((has_single_bit(TyAllocSize) &&
3013 match(GEP.getOperand(1),
3014 m_Shr(m_Value(V),
3015 m_SpecificInt(countr_zero(TyAllocSize))))) ||
3016 match(GEP.getOperand(1),
3017 m_IDiv(m_Value(V), m_SpecificInt(TyAllocSize)))) {
3019 GEP.getPointerOperand(), V,
3020 GEP.getNoWrapFlags());
3021 }
3022 }
3023 if (ExactIns->isExact() && ExactIns->hasOneUse()) {
3024 // Try to canonicalize non-i8 element type to i8 if the index is an
3025 // exact instruction. If the index is an exact instruction (div/shr)
3026 // with a constant RHS, we can fold the non-i8 element scale into the
3027 // div/shr (similiar to the mul case, just inverted).
3028 const APInt *C;
3029 std::optional<APInt> NewC;
3030 if (has_single_bit(TyAllocSize) &&
3031 match(ExactIns, m_Shr(m_Value(V), m_APInt(C))) &&
3032 C->uge(countr_zero(TyAllocSize)))
3033 NewC = *C - countr_zero(TyAllocSize);
3034 else if (match(ExactIns, m_UDiv(m_Value(V), m_APInt(C)))) {
3035 APInt Quot;
3036 uint64_t Rem;
3037 APInt::udivrem(*C, TyAllocSize, Quot, Rem);
3038 if (Rem == 0)
3039 NewC = Quot;
3040 } else if (match(ExactIns, m_SDiv(m_Value(V), m_APInt(C)))) {
3041 APInt Quot;
3042 int64_t Rem;
3043 APInt::sdivrem(*C, TyAllocSize, Quot, Rem);
3044 // For sdiv we need to make sure we arent creating INT_MIN / -1.
3045 if (!Quot.isAllOnes() && Rem == 0)
3046 NewC = Quot;
3047 }
3048
3049 if (NewC.has_value()) {
3050 Value *NewOp = Builder.CreateBinOp(
3051 static_cast<Instruction::BinaryOps>(ExactIns->getOpcode()), V,
3052 ConstantInt::get(V->getType(), *NewC));
3053 cast<BinaryOperator>(NewOp)->setIsExact();
3055 GEP.getPointerOperand(), NewOp,
3056 GEP.getNoWrapFlags());
3057 }
3058 }
3059 }
3060 }
3061 }
3062 // We do not handle pointer-vector geps here.
3063 if (GEPType->isVectorTy())
3064 return nullptr;
3065
3066 if (GEP.getNumIndices() == 1) {
3067 // We can only preserve inbounds if the original gep is inbounds, the add
3068 // is nsw, and the add operands are non-negative.
3069 auto CanPreserveInBounds = [&](bool AddIsNSW, Value *Idx1, Value *Idx2) {
3071 return GEP.isInBounds() && AddIsNSW && isKnownNonNegative(Idx1, Q) &&
3072 isKnownNonNegative(Idx2, Q);
3073 };
3074
3075 // Try to replace ADD + GEP with GEP + GEP.
3076 Value *Idx1, *Idx2;
3077 if (match(GEP.getOperand(1),
3078 m_OneUse(m_Add(m_Value(Idx1), m_Value(Idx2))))) {
3079 // %idx = add i64 %idx1, %idx2
3080 // %gep = getelementptr i32, ptr %ptr, i64 %idx
3081 // as:
3082 // %newptr = getelementptr i32, ptr %ptr, i64 %idx1
3083 // %newgep = getelementptr i32, ptr %newptr, i64 %idx2
3084 bool IsInBounds = CanPreserveInBounds(
3085 cast<OverflowingBinaryOperator>(GEP.getOperand(1))->hasNoSignedWrap(),
3086 Idx1, Idx2);
3087 auto *NewPtr =
3088 Builder.CreateGEP(GEP.getSourceElementType(), GEP.getPointerOperand(),
3089 Idx1, "", IsInBounds);
3090 return replaceInstUsesWith(
3091 GEP, Builder.CreateGEP(GEP.getSourceElementType(), NewPtr, Idx2, "",
3092 IsInBounds));
3093 }
3094 ConstantInt *C;
3095 if (match(GEP.getOperand(1), m_OneUse(m_SExtLike(m_OneUse(m_NSWAdd(
3096 m_Value(Idx1), m_ConstantInt(C))))))) {
3097 // %add = add nsw i32 %idx1, idx2
3098 // %sidx = sext i32 %add to i64
3099 // %gep = getelementptr i32, ptr %ptr, i64 %sidx
3100 // as:
3101 // %newptr = getelementptr i32, ptr %ptr, i32 %idx1
3102 // %newgep = getelementptr i32, ptr %newptr, i32 idx2
3103 bool IsInBounds = CanPreserveInBounds(
3104 /*IsNSW=*/true, Idx1, C);
3105 auto *NewPtr = Builder.CreateGEP(
3106 GEP.getSourceElementType(), GEP.getPointerOperand(),
3107 Builder.CreateSExt(Idx1, GEP.getOperand(1)->getType()), "",
3108 IsInBounds);
3109 return replaceInstUsesWith(
3110 GEP,
3111 Builder.CreateGEP(GEP.getSourceElementType(), NewPtr,
3112 Builder.CreateSExt(C, GEP.getOperand(1)->getType()),
3113 "", IsInBounds));
3114 }
3115 }
3116
3117 if (!GEP.isInBounds()) {
3118 unsigned IdxWidth =
3120 APInt BasePtrOffset(IdxWidth, 0);
3121 Value *UnderlyingPtrOp =
3123 BasePtrOffset);
3124 bool CanBeNull, CanBeFreed;
3125 uint64_t DerefBytes = UnderlyingPtrOp->getPointerDereferenceableBytes(
3126 DL, CanBeNull, CanBeFreed);
3127 if (!CanBeNull && !CanBeFreed && DerefBytes != 0) {
3128 if (GEP.accumulateConstantOffset(DL, BasePtrOffset) &&
3129 BasePtrOffset.isNonNegative()) {
3130 APInt AllocSize(IdxWidth, DerefBytes);
3131 if (BasePtrOffset.ule(AllocSize)) {
3133 GEP.getSourceElementType(), PtrOp, Indices, GEP.getName());
3134 }
3135 }
3136 }
3137 }
3138
3139 // nusw + nneg -> nuw
3140 if (GEP.hasNoUnsignedSignedWrap() && !GEP.hasNoUnsignedWrap() &&
3141 all_of(GEP.indices(), [&](Value *Idx) {
3142 return isKnownNonNegative(Idx, SQ.getWithInstruction(&GEP));
3143 })) {
3144 GEP.setNoWrapFlags(GEP.getNoWrapFlags() | GEPNoWrapFlags::noUnsignedWrap());
3145 return &GEP;
3146 }
3147
3149 return R;
3150
3151 return nullptr;
3152}
3153
3155 Instruction *AI) {
3156 if (isa<ConstantPointerNull>(V))
3157 return true;
3158 if (auto *LI = dyn_cast<LoadInst>(V))
3159 return isa<GlobalVariable>(LI->getPointerOperand());
3160 // Two distinct allocations will never be equal.
3161 return isAllocLikeFn(V, &TLI) && V != AI;
3162}
3163
3164/// Given a call CB which uses an address UsedV, return true if we can prove the
3165/// call's only possible effect is storing to V.
3166static bool isRemovableWrite(CallBase &CB, Value *UsedV,
3167 const TargetLibraryInfo &TLI) {
3168 if (!CB.use_empty())
3169 // TODO: add recursion if returned attribute is present
3170 return false;
3171
3172 if (CB.isTerminator())
3173 // TODO: remove implementation restriction
3174 return false;
3175
3176 if (!CB.willReturn() || !CB.doesNotThrow())
3177 return false;
3178
3179 // If the only possible side effect of the call is writing to the alloca,
3180 // and the result isn't used, we can safely remove any reads implied by the
3181 // call including those which might read the alloca itself.
3182 std::optional<MemoryLocation> Dest = MemoryLocation::getForDest(&CB, TLI);
3183 return Dest && Dest->Ptr == UsedV;
3184}
3185
3188 const TargetLibraryInfo &TLI) {
3190 const std::optional<StringRef> Family = getAllocationFamily(AI, &TLI);
3191 Worklist.push_back(AI);
3192
3193 do {
3194 Instruction *PI = Worklist.pop_back_val();
3195 for (User *U : PI->users()) {
3196 Instruction *I = cast<Instruction>(U);
3197 switch (I->getOpcode()) {
3198 default:
3199 // Give up the moment we see something we can't handle.
3200 return false;
3201
3202 case Instruction::AddrSpaceCast:
3203 case Instruction::BitCast:
3204 case Instruction::GetElementPtr:
3205 Users.emplace_back(I);
3206 Worklist.push_back(I);
3207 continue;
3208
3209 case Instruction::ICmp: {
3210 ICmpInst *ICI = cast<ICmpInst>(I);
3211 // We can fold eq/ne comparisons with null to false/true, respectively.
3212 // We also fold comparisons in some conditions provided the alloc has
3213 // not escaped (see isNeverEqualToUnescapedAlloc).
3214 if (!ICI->isEquality())
3215 return false;
3216 unsigned OtherIndex = (ICI->getOperand(0) == PI) ? 1 : 0;
3217 if (!isNeverEqualToUnescapedAlloc(ICI->getOperand(OtherIndex), TLI, AI))
3218 return false;
3219
3220 // Do not fold compares to aligned_alloc calls, as they may have to
3221 // return null in case the required alignment cannot be satisfied,
3222 // unless we can prove that both alignment and size are valid.
3223 auto AlignmentAndSizeKnownValid = [](CallBase *CB) {
3224 // Check if alignment and size of a call to aligned_alloc is valid,
3225 // that is alignment is a power-of-2 and the size is a multiple of the
3226 // alignment.
3227 const APInt *Alignment;
3228 const APInt *Size;
3229 return match(CB->getArgOperand(0), m_APInt(Alignment)) &&
3230 match(CB->getArgOperand(1), m_APInt(Size)) &&
3231 Alignment->isPowerOf2() && Size->urem(*Alignment).isZero();
3232 };
3233 auto *CB = dyn_cast<CallBase>(AI);
3234 LibFunc TheLibFunc;
3235 if (CB && TLI.getLibFunc(*CB->getCalledFunction(), TheLibFunc) &&
3236 TLI.has(TheLibFunc) && TheLibFunc == LibFunc_aligned_alloc &&
3237 !AlignmentAndSizeKnownValid(CB))
3238 return false;
3239 Users.emplace_back(I);
3240 continue;
3241 }
3242
3243 case Instruction::Call:
3244 // Ignore no-op and store intrinsics.
3245 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
3246 switch (II->getIntrinsicID()) {
3247 default:
3248 return false;
3249
3250 case Intrinsic::memmove:
3251 case Intrinsic::memcpy:
3252 case Intrinsic::memset: {
3253 MemIntrinsic *MI = cast<MemIntrinsic>(II);
3254 if (MI->isVolatile() || MI->getRawDest() != PI)
3255 return false;
3256 [[fallthrough]];
3257 }
3258 case Intrinsic::assume:
3259 case Intrinsic::invariant_start:
3260 case Intrinsic::invariant_end:
3261 case Intrinsic::lifetime_start:
3262 case Intrinsic::lifetime_end:
3263 case Intrinsic::objectsize:
3264 Users.emplace_back(I);
3265 continue;
3266 case Intrinsic::launder_invariant_group:
3267 case Intrinsic::strip_invariant_group:
3268 Users.emplace_back(I);
3269 Worklist.push_back(I);
3270 continue;
3271 }
3272 }
3273
3274 if (isRemovableWrite(*cast<CallBase>(I), PI, TLI)) {
3275 Users.emplace_back(I);
3276 continue;
3277 }
3278
3279 if (getFreedOperand(cast<CallBase>(I), &TLI) == PI &&
3280 getAllocationFamily(I, &TLI) == Family) {
3281 assert(Family);
3282 Users.emplace_back(I);
3283 continue;
3284 }
3285
3286 if (getReallocatedOperand(cast<CallBase>(I)) == PI &&
3287 getAllocationFamily(I, &TLI) == Family) {
3288 assert(Family);
3289 Users.emplace_back(I);
3290 Worklist.push_back(I);
3291 continue;
3292 }
3293
3294 return false;
3295
3296 case Instruction::Store: {
3297 StoreInst *SI = cast<StoreInst>(I);
3298 if (SI->isVolatile() || SI->getPointerOperand() != PI)
3299 return false;
3300 Users.emplace_back(I);
3301 continue;
3302 }
3303 }
3304 llvm_unreachable("missing a return?");
3305 }
3306 } while (!Worklist.empty());
3307 return true;
3308}
3309
3311 assert(isa<AllocaInst>(MI) || isRemovableAlloc(&cast<CallBase>(MI), &TLI));
3312
3313 // If we have a malloc call which is only used in any amount of comparisons to
3314 // null and free calls, delete the calls and replace the comparisons with true
3315 // or false as appropriate.
3316
3317 // This is based on the principle that we can substitute our own allocation
3318 // function (which will never return null) rather than knowledge of the
3319 // specific function being called. In some sense this can change the permitted
3320 // outputs of a program (when we convert a malloc to an alloca, the fact that
3321 // the allocation is now on the stack is potentially visible, for example),
3322 // but we believe in a permissible manner.
3324
3325 // If we are removing an alloca with a dbg.declare, insert dbg.value calls
3326 // before each store.
3329 std::unique_ptr<DIBuilder> DIB;
3330 if (isa<AllocaInst>(MI)) {
3331 findDbgUsers(DVIs, &MI, &DVRs);
3332 DIB.reset(new DIBuilder(*MI.getModule(), /*AllowUnresolved=*/false));
3333 }
3334
3335 if (isAllocSiteRemovable(&MI, Users, TLI)) {
3336 for (unsigned i = 0, e = Users.size(); i != e; ++i) {
3337 // Lowering all @llvm.objectsize calls first because they may
3338 // use a bitcast/GEP of the alloca we are removing.
3339 if (!Users[i])
3340 continue;
3341
3342 Instruction *I = cast<Instruction>(&*Users[i]);
3343
3344 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
3345 if (II->getIntrinsicID() == Intrinsic::objectsize) {
3346 SmallVector<Instruction *> InsertedInstructions;
3347 Value *Result = lowerObjectSizeCall(
3348 II, DL, &TLI, AA, /*MustSucceed=*/true, &InsertedInstructions);
3349 for (Instruction *Inserted : InsertedInstructions)
3350 Worklist.add(Inserted);
3351 replaceInstUsesWith(*I, Result);
3353 Users[i] = nullptr; // Skip examining in the next loop.
3354 }
3355 }
3356 }
3357 for (unsigned i = 0, e = Users.size(); i != e; ++i) {
3358 if (!Users[i])
3359 continue;
3360
3361 Instruction *I = cast<Instruction>(&*Users[i]);
3362
3363 if (ICmpInst *C = dyn_cast<ICmpInst>(I)) {
3365 ConstantInt::get(Type::getInt1Ty(C->getContext()),
3366 C->isFalseWhenEqual()));
3367 } else if (auto *SI = dyn_cast<StoreInst>(I)) {
3368 for (auto *DVI : DVIs)
3369 if (DVI->isAddressOfVariable())
3370 ConvertDebugDeclareToDebugValue(DVI, SI, *DIB);
3371 for (auto *DVR : DVRs)
3372 if (DVR->isAddressOfVariable())
3373 ConvertDebugDeclareToDebugValue(DVR, SI, *DIB);
3374 } else {
3375 // Casts, GEP, or anything else: we're about to delete this instruction,
3376 // so it can not have any valid uses.
3377 replaceInstUsesWith(*I, PoisonValue::get(I->getType()));
3378 }
3380 }
3381
3382 if (InvokeInst *II = dyn_cast<InvokeInst>(&MI)) {
3383 // Replace invoke with a NOP intrinsic to maintain the original CFG
3384 Module *M = II->getModule();
3385 Function *F = Intrinsic::getOrInsertDeclaration(M, Intrinsic::donothing);
3386 InvokeInst::Create(F, II->getNormalDest(), II->getUnwindDest(), {}, "",
3387 II->getParent());
3388 }
3389
3390 // Remove debug intrinsics which describe the value contained within the
3391 // alloca. In addition to removing dbg.{declare,addr} which simply point to
3392 // the alloca, remove dbg.value(<alloca>, ..., DW_OP_deref)'s as well, e.g.:
3393 //
3394 // ```
3395 // define void @foo(i32 %0) {
3396 // %a = alloca i32 ; Deleted.
3397 // store i32 %0, i32* %a
3398 // dbg.value(i32 %0, "arg0") ; Not deleted.
3399 // dbg.value(i32* %a, "arg0", DW_OP_deref) ; Deleted.
3400 // call void @trivially_inlinable_no_op(i32* %a)
3401 // ret void
3402 // }
3403 // ```
3404 //
3405 // This may not be required if we stop describing the contents of allocas
3406 // using dbg.value(<alloca>, ..., DW_OP_deref), but we currently do this in
3407 // the LowerDbgDeclare utility.
3408 //
3409 // If there is a dead store to `%a` in @trivially_inlinable_no_op, the
3410 // "arg0" dbg.value may be stale after the call. However, failing to remove
3411 // the DW_OP_deref dbg.value causes large gaps in location coverage.
3412 //
3413 // FIXME: the Assignment Tracking project has now likely made this
3414 // redundant (and it's sometimes harmful).
3415 for (auto *DVI : DVIs)
3416 if (DVI->isAddressOfVariable() || DVI->getExpression()->startsWithDeref())
3417 DVI->eraseFromParent();
3418 for (auto *DVR : DVRs)
3419 if (DVR->isAddressOfVariable() || DVR->getExpression()->startsWithDeref())
3420 DVR->eraseFromParent();
3421
3422 return eraseInstFromFunction(MI);
3423 }
3424 return nullptr;
3425}
3426
3427/// Move the call to free before a NULL test.
3428///
3429/// Check if this free is accessed after its argument has been test
3430/// against NULL (property 0).
3431/// If yes, it is legal to move this call in its predecessor block.
3432///
3433/// The move is performed only if the block containing the call to free
3434/// will be removed, i.e.:
3435/// 1. it has only one predecessor P, and P has two successors
3436/// 2. it contains the call, noops, and an unconditional branch
3437/// 3. its successor is the same as its predecessor's successor
3438///
3439/// The profitability is out-of concern here and this function should
3440/// be called only if the caller knows this transformation would be
3441/// profitable (e.g., for code size).
3443 const DataLayout &DL) {
3444 Value *Op = FI.getArgOperand(0);
3445 BasicBlock *FreeInstrBB = FI.getParent();
3446 BasicBlock *PredBB = FreeInstrBB->getSinglePredecessor();
3447
3448 // Validate part of constraint #1: Only one predecessor
3449 // FIXME: We can extend the number of predecessor, but in that case, we
3450 // would duplicate the call to free in each predecessor and it may
3451 // not be profitable even for code size.
3452 if (!PredBB)
3453 return nullptr;
3454
3455 // Validate constraint #2: Does this block contains only the call to
3456 // free, noops, and an unconditional branch?
3457 BasicBlock *SuccBB;
3458 Instruction *FreeInstrBBTerminator = FreeInstrBB->getTerminator();
3459 if (!match(FreeInstrBBTerminator, m_UnconditionalBr(SuccBB)))
3460 return nullptr;
3461
3462 // If there are only 2 instructions in the block, at this point,
3463 // this is the call to free and unconditional.
3464 // If there are more than 2 instructions, check that they are noops
3465 // i.e., they won't hurt the performance of the generated code.
3466 if (FreeInstrBB->size() != 2) {
3467 for (const Instruction &Inst : FreeInstrBB->instructionsWithoutDebug()) {
3468 if (&Inst == &FI || &Inst == FreeInstrBBTerminator)
3469 continue;
3470 auto *Cast = dyn_cast<CastInst>(&Inst);
3471 if (!Cast || !Cast->isNoopCast(DL))
3472 return nullptr;
3473 }
3474 }
3475 // Validate the rest of constraint #1 by matching on the pred branch.
3476 Instruction *TI = PredBB->getTerminator();
3477 BasicBlock *TrueBB, *FalseBB;
3478 CmpPredicate Pred;
3479 if (!match(TI, m_Br(m_ICmp(Pred,
3481 m_Specific(Op->stripPointerCasts())),
3482 m_Zero()),
3483 TrueBB, FalseBB)))
3484 return nullptr;
3485 if (Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
3486 return nullptr;
3487
3488 // Validate constraint #3: Ensure the null case just falls through.
3489 if (SuccBB != (Pred == ICmpInst::ICMP_EQ ? TrueBB : FalseBB))
3490 return nullptr;
3491 assert(FreeInstrBB == (Pred == ICmpInst::ICMP_EQ ? FalseBB : TrueBB) &&
3492 "Broken CFG: missing edge from predecessor to successor");
3493
3494 // At this point, we know that everything in FreeInstrBB can be moved
3495 // before TI.
3496 for (Instruction &Instr : llvm::make_early_inc_range(*FreeInstrBB)) {
3497 if (&Instr == FreeInstrBBTerminator)
3498 break;
3499 Instr.moveBeforePreserving(TI);
3500 }
3501 assert(FreeInstrBB->size() == 1 &&
3502 "Only the branch instruction should remain");
3503
3504 // Now that we've moved the call to free before the NULL check, we have to
3505 // remove any attributes on its parameter that imply it's non-null, because
3506 // those attributes might have only been valid because of the NULL check, and
3507 // we can get miscompiles if we keep them. This is conservative if non-null is
3508 // also implied by something other than the NULL check, but it's guaranteed to
3509 // be correct, and the conservativeness won't matter in practice, since the
3510 // attributes are irrelevant for the call to free itself and the pointer
3511 // shouldn't be used after the call.
3512 AttributeList Attrs = FI.getAttributes();
3513 Attrs = Attrs.removeParamAttribute(FI.getContext(), 0, Attribute::NonNull);
3514 Attribute Dereferenceable = Attrs.getParamAttr(0, Attribute::Dereferenceable);
3515 if (Dereferenceable.isValid()) {
3516 uint64_t Bytes = Dereferenceable.getDereferenceableBytes();
3517 Attrs = Attrs.removeParamAttribute(FI.getContext(), 0,
3518 Attribute::Dereferenceable);
3519 Attrs = Attrs.addDereferenceableOrNullParamAttr(FI.getContext(), 0, Bytes);
3520 }
3521 FI.setAttributes(Attrs);
3522
3523 return &FI;
3524}
3525
3527 // free undef -> unreachable.
3528 if (isa<UndefValue>(Op)) {
3529 // Leave a marker since we can't modify the CFG here.
3531 return eraseInstFromFunction(FI);
3532 }
3533
3534 // If we have 'free null' delete the instruction. This can happen in stl code
3535 // when lots of inlining happens.
3536 if (isa<ConstantPointerNull>(Op))
3537 return eraseInstFromFunction(FI);
3538
3539 // If we had free(realloc(...)) with no intervening uses, then eliminate the
3540 // realloc() entirely.
3541 CallInst *CI = dyn_cast<CallInst>(Op);
3542 if (CI && CI->hasOneUse())
3543 if (Value *ReallocatedOp = getReallocatedOperand(CI))
3544 return eraseInstFromFunction(*replaceInstUsesWith(*CI, ReallocatedOp));
3545
3546 // If we optimize for code size, try to move the call to free before the null
3547 // test so that simplify cfg can remove the empty block and dead code
3548 // elimination the branch. I.e., helps to turn something like:
3549 // if (foo) free(foo);
3550 // into
3551 // free(foo);
3552 //
3553 // Note that we can only do this for 'free' and not for any flavor of
3554 // 'operator delete'; there is no 'operator delete' symbol for which we are
3555 // permitted to invent a call, even if we're passing in a null pointer.
3556 if (MinimizeSize) {
3557 LibFunc Func;
3558 if (TLI.getLibFunc(FI, Func) && TLI.has(Func) && Func == LibFunc_free)
3560 return I;
3561 }
3562
3563 return nullptr;
3564}
3565
3567 Value *RetVal = RI.getReturnValue();
3568 if (!RetVal || !AttributeFuncs::isNoFPClassCompatibleType(RetVal->getType()))
3569 return nullptr;
3570
3571 Function *F = RI.getFunction();
3572 FPClassTest ReturnClass = F->getAttributes().getRetNoFPClass();
3573 if (ReturnClass == fcNone)
3574 return nullptr;
3575
3576 KnownFPClass KnownClass;
3577 Value *Simplified =
3578 SimplifyDemandedUseFPClass(RetVal, ~ReturnClass, KnownClass, 0, &RI);
3579 if (!Simplified)
3580 return nullptr;
3581
3582 return ReturnInst::Create(RI.getContext(), Simplified);
3583}
3584
3585// WARNING: keep in sync with SimplifyCFGOpt::simplifyUnreachable()!
3587 // Try to remove the previous instruction if it must lead to unreachable.
3588 // This includes instructions like stores and "llvm.assume" that may not get
3589 // removed by simple dead code elimination.
3590 bool Changed = false;
3591 while (Instruction *Prev = I.getPrevNonDebugInstruction()) {
3592 // While we theoretically can erase EH, that would result in a block that
3593 // used to start with an EH no longer starting with EH, which is invalid.
3594 // To make it valid, we'd need to fixup predecessors to no longer refer to
3595 // this block, but that changes CFG, which is not allowed in InstCombine.
3596 if (Prev->isEHPad())
3597 break; // Can not drop any more instructions. We're done here.
3598
3600 break; // Can not drop any more instructions. We're done here.
3601 // Otherwise, this instruction can be freely erased,
3602 // even if it is not side-effect free.
3603
3604 // A value may still have uses before we process it here (for example, in
3605 // another unreachable block), so convert those to poison.
3606 replaceInstUsesWith(*Prev, PoisonValue::get(Prev->getType()));
3607 eraseInstFromFunction(*Prev);
3608 Changed = true;
3609 }
3610 return Changed;
3611}
3612
3615 return nullptr;
3616}
3617
3619 assert(BI.isUnconditional() && "Only for unconditional branches.");
3620
3621 // If this store is the second-to-last instruction in the basic block
3622 // (excluding debug info and bitcasts of pointers) and if the block ends with
3623 // an unconditional branch, try to move the store to the successor block.
3624
3625 auto GetLastSinkableStore = [](BasicBlock::iterator BBI) {
3626 auto IsNoopInstrForStoreMerging = [](BasicBlock::iterator BBI) {
3627 return BBI->isDebugOrPseudoInst() ||
3628 (isa<BitCastInst>(BBI) && BBI->getType()->isPointerTy());
3629 };
3630
3631 BasicBlock::iterator FirstInstr = BBI->getParent()->begin();
3632 do {
3633 if (BBI != FirstInstr)
3634 --BBI;
3635 } while (BBI != FirstInstr && IsNoopInstrForStoreMerging(BBI));
3636
3637 return dyn_cast<StoreInst>(BBI);
3638 };
3639
3640 if (StoreInst *SI = GetLastSinkableStore(BasicBlock::iterator(BI)))
3641 if (mergeStoreIntoSuccessor(*SI))
3642 return &BI;
3643
3644 return nullptr;
3645}
3646
3649 if (!DeadEdges.insert({From, To}).second)
3650 return;
3651
3652 // Replace phi node operands in successor with poison.
3653 for (PHINode &PN : To->phis())
3654 for (Use &U : PN.incoming_values())
3655 if (PN.getIncomingBlock(U) == From && !isa<PoisonValue>(U)) {
3656 replaceUse(U, PoisonValue::get(PN.getType()));
3657 addToWorklist(&PN);
3658 MadeIRChange = true;
3659 }
3660
3661 Worklist.push_back(To);
3662}
3663
3664// Under the assumption that I is unreachable, remove it and following
3665// instructions. Changes are reported directly to MadeIRChange.
3668 BasicBlock *BB = I->getParent();
3669 for (Instruction &Inst : make_early_inc_range(
3670 make_range(std::next(BB->getTerminator()->getReverseIterator()),
3671 std::next(I->getReverseIterator())))) {
3672 if (!Inst.use_empty() && !Inst.getType()->isTokenTy()) {
3673 replaceInstUsesWith(Inst, PoisonValue::get(Inst.getType()));
3674 MadeIRChange = true;
3675 }
3676 if (Inst.isEHPad() || Inst.getType()->isTokenTy())
3677 continue;
3678 // RemoveDIs: erase debug-info on this instruction manually.
3679 Inst.dropDbgRecords();
3681 MadeIRChange = true;
3682 }
3683
3684 SmallVector<Value *> Changed;
3685 if (handleUnreachableTerminator(BB->getTerminator(), Changed)) {
3686 MadeIRChange = true;
3687 for (Value *V : Changed)
3688 addToWorklist(cast<Instruction>(V));
3689 }
3690
3691 // Handle potentially dead successors.
3692 for (BasicBlock *Succ : successors(BB))
3693 addDeadEdge(BB, Succ, Worklist);
3694}
3695
3698 while (!Worklist.empty()) {
3699 BasicBlock *BB = Worklist.pop_back_val();
3700 if (!all_of(predecessors(BB), [&](BasicBlock *Pred) {
3701 return DeadEdges.contains({Pred, BB}) || DT.dominates(BB, Pred);
3702 }))
3703 continue;
3704
3706 }
3707}
3708
3710 BasicBlock *LiveSucc) {
3712 for (BasicBlock *Succ : successors(BB)) {
3713 // The live successor isn't dead.
3714 if (Succ == LiveSucc)
3715 continue;
3716
3717 addDeadEdge(BB, Succ, Worklist);
3718 }
3719
3721}
3722
3724 if (BI.isUnconditional())
3726
3727 // Change br (not X), label True, label False to: br X, label False, True
3728 Value *Cond = BI.getCondition();
3729 Value *X;
3730 if (match(Cond, m_Not(m_Value(X))) && !isa<Constant>(X)) {
3731 // Swap Destinations and condition...
3732 BI.swapSuccessors();
3733 if (BPI)
3735 return replaceOperand(BI, 0, X);
3736 }
3737
3738 // Canonicalize logical-and-with-invert as logical-or-with-invert.
3739 // This is done by inverting the condition and swapping successors:
3740 // br (X && !Y), T, F --> br !(X && !Y), F, T --> br (!X || Y), F, T
3741 Value *Y;
3742 if (isa<SelectInst>(Cond) &&
3743 match(Cond,
3745 Value *NotX = Builder.CreateNot(X, "not." + X->getName());
3746 Value *Or = Builder.CreateLogicalOr(NotX, Y);
3747 BI.swapSuccessors();
3748 if (BPI)
3750 return replaceOperand(BI, 0, Or);
3751 }
3752
3753 // If the condition is irrelevant, remove the use so that other
3754 // transforms on the condition become more effective.
3755 if (!isa<ConstantInt>(Cond) && BI.getSuccessor(0) == BI.getSuccessor(1))
3756 return replaceOperand(BI, 0, ConstantInt::getFalse(Cond->getType()));
3757
3758 // Canonicalize, for example, fcmp_one -> fcmp_oeq.
3759 CmpPredicate Pred;
3760 if (match(Cond, m_OneUse(m_FCmp(Pred, m_Value(), m_Value()))) &&
3761 !isCanonicalPredicate(Pred)) {
3762 // Swap destinations and condition.
3763 auto *Cmp = cast<CmpInst>(Cond);
3764 Cmp->setPredicate(CmpInst::getInversePredicate(Pred));
3765 BI.swapSuccessors();
3766 if (BPI)
3768 Worklist.push(Cmp);
3769 return &BI;
3770 }
3771
3772 if (isa<UndefValue>(Cond)) {
3773 handlePotentiallyDeadSuccessors(BI.getParent(), /*LiveSucc*/ nullptr);
3774 return nullptr;
3775 }
3776 if (auto *CI = dyn_cast<ConstantInt>(Cond)) {
3778 BI.getSuccessor(!CI->getZExtValue()));
3779 return nullptr;
3780 }
3781
3782 // Replace all dominated uses of the condition with true/false
3783 // Ignore constant expressions to avoid iterating over uses on other
3784 // functions.
3785 if (!isa<Constant>(Cond) && BI.getSuccessor(0) != BI.getSuccessor(1)) {
3786 for (auto &U : make_early_inc_range(Cond->uses())) {
3787 BasicBlockEdge Edge0(BI.getParent(), BI.getSuccessor(0));
3788 if (DT.dominates(Edge0, U)) {
3789 replaceUse(U, ConstantInt::getTrue(Cond->getType()));
3790 addToWorklist(cast<Instruction>(U.getUser()));
3791 continue;
3792 }
3793 BasicBlockEdge Edge1(BI.getParent(), BI.getSuccessor(1));
3794 if (DT.dominates(Edge1, U)) {
3795 replaceUse(U, ConstantInt::getFalse(Cond->getType()));
3796 addToWorklist(cast<Instruction>(U.getUser()));
3797 }
3798 }
3799 }
3800
3801 DC.registerBranch(&BI);
3802 return nullptr;
3803}
3804
3805// Replaces (switch (select cond, X, C)/(select cond, C, X)) with (switch X) if
3806// we can prove that both (switch C) and (switch X) go to the default when cond
3807// is false/true.
3810 bool IsTrueArm) {
3811 unsigned CstOpIdx = IsTrueArm ? 1 : 2;
3812 auto *C = dyn_cast<ConstantInt>(Select->getOperand(CstOpIdx));
3813 if (!C)
3814 return nullptr;
3815
3816 BasicBlock *CstBB = SI.findCaseValue(C)->getCaseSuccessor();
3817 if (CstBB != SI.getDefaultDest())
3818 return nullptr;
3819 Value *X = Select->getOperand(3 - CstOpIdx);
3820 CmpPredicate Pred;
3821 const APInt *RHSC;
3822 if (!match(Select->getCondition(),
3823 m_ICmp(Pred, m_Specific(X), m_APInt(RHSC))))
3824 return nullptr;
3825 if (IsTrueArm)
3826 Pred = ICmpInst::getInversePredicate(Pred);
3827
3828 // See whether we can replace the select with X
3830 for (auto Case : SI.cases())
3831 if (!CR.contains(Case.getCaseValue()->getValue()))
3832 return nullptr;
3833
3834 return X;
3835}
3836
3838 Value *Cond = SI.getCondition();
3839 Value *Op0;
3840 ConstantInt *AddRHS;
3841 if (match(Cond, m_Add(m_Value(Op0), m_ConstantInt(AddRHS)))) {
3842 // Change 'switch (X+4) case 1:' into 'switch (X) case -3'.
3843 for (auto Case : SI.cases()) {
3844 Constant *NewCase = ConstantExpr::getSub(Case.getCaseValue(), AddRHS);
3845 assert(isa<ConstantInt>(NewCase) &&
3846 "Result of expression should be constant");
3847 Case.setValue(cast<ConstantInt>(NewCase));
3848 }
3849 return replaceOperand(SI, 0, Op0);
3850 }
3851
3852 ConstantInt *SubLHS;
3853 if (match(Cond, m_Sub(m_ConstantInt(SubLHS), m_Value(Op0)))) {
3854 // Change 'switch (1-X) case 1:' into 'switch (X) case 0'.
3855 for (auto Case : SI.cases()) {
3856 Constant *NewCase = ConstantExpr::getSub(SubLHS, Case.getCaseValue());
3857 assert(isa<ConstantInt>(NewCase) &&
3858 "Result of expression should be constant");
3859 Case.setValue(cast<ConstantInt>(NewCase));
3860 }
3861 return replaceOperand(SI, 0, Op0);
3862 }
3863
3864 uint64_t ShiftAmt;
3865 if (match(Cond, m_Shl(m_Value(Op0), m_ConstantInt(ShiftAmt))) &&
3866 ShiftAmt < Op0->getType()->getScalarSizeInBits() &&
3867 all_of(SI.cases(), [&](const auto &Case) {
3868 return Case.getCaseValue()->getValue().countr_zero() >= ShiftAmt;
3869 })) {
3870 // Change 'switch (X << 2) case 4:' into 'switch (X) case 1:'.
3871 OverflowingBinaryOperator *Shl = cast<OverflowingBinaryOperator>(Cond);
3872 if (Shl->hasNoUnsignedWrap() || Shl->hasNoSignedWrap() ||
3873 Shl->hasOneUse()) {
3874 Value *NewCond = Op0;
3875 if (!Shl->hasNoUnsignedWrap() && !Shl->hasNoSignedWrap()) {
3876 // If the shift may wrap, we need to mask off the shifted bits.
3877 unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
3878 NewCond = Builder.CreateAnd(
3879 Op0, APInt::getLowBitsSet(BitWidth, BitWidth - ShiftAmt));
3880 }
3881 for (auto Case : SI.cases()) {
3882 const APInt &CaseVal = Case.getCaseValue()->getValue();
3883 APInt ShiftedCase = Shl->hasNoSignedWrap() ? CaseVal.ashr(ShiftAmt)
3884 : CaseVal.lshr(ShiftAmt);
3885 Case.setValue(ConstantInt::get(SI.getContext(), ShiftedCase));
3886 }
3887 return replaceOperand(SI, 0, NewCond);
3888 }
3889 }
3890
3891 // Fold switch(zext/sext(X)) into switch(X) if possible.
3892 if (match(Cond, m_ZExtOrSExt(m_Value(Op0)))) {
3893 bool IsZExt = isa<ZExtInst>(Cond);
3894 Type *SrcTy = Op0->getType();
3895 unsigned NewWidth = SrcTy->getScalarSizeInBits();
3896
3897 if (all_of(SI.cases(), [&](const auto &Case) {
3898 const APInt &CaseVal = Case.getCaseValue()->getValue();
3899 return IsZExt ? CaseVal.isIntN(NewWidth)
3900 : CaseVal.isSignedIntN(NewWidth);
3901 })) {
3902 for (auto &Case : SI.cases()) {
3903 APInt TruncatedCase = Case.getCaseValue()->getValue().trunc(NewWidth);
3904 Case.setValue(ConstantInt::get(SI.getContext(), TruncatedCase));
3905 }
3906 return replaceOperand(SI, 0, Op0);
3907 }
3908 }
3909
3910 // Fold switch(select cond, X, Y) into switch(X/Y) if possible
3911 if (auto *Select = dyn_cast<SelectInst>(Cond)) {
3912 if (Value *V =
3913 simplifySwitchOnSelectUsingRanges(SI, Select, /*IsTrueArm=*/true))
3914 return replaceOperand(SI, 0, V);
3915 if (Value *V =
3916 simplifySwitchOnSelectUsingRanges(SI, Select, /*IsTrueArm=*/false))
3917 return replaceOperand(SI, 0, V);
3918 }
3919
3920 KnownBits Known = computeKnownBits(Cond, 0, &SI);
3921 unsigned LeadingKnownZeros = Known.countMinLeadingZeros();
3922 unsigned LeadingKnownOnes = Known.countMinLeadingOnes();
3923
3924 // Compute the number of leading bits we can ignore.
3925 // TODO: A better way to determine this would use ComputeNumSignBits().
3926 for (const auto &C : SI.cases()) {
3927 LeadingKnownZeros =
3928 std::min(LeadingKnownZeros, C.getCaseValue()->getValue().countl_zero());
3929 LeadingKnownOnes =
3930 std::min(LeadingKnownOnes, C.getCaseValue()->getValue().countl_one());
3931 }
3932
3933 unsigned NewWidth = Known.getBitWidth() - std::max(LeadingKnownZeros, LeadingKnownOnes);
3934
3935 // Shrink the condition operand if the new type is smaller than the old type.
3936 // But do not shrink to a non-standard type, because backend can't generate
3937 // good code for that yet.
3938 // TODO: We can make it aggressive again after fixing PR39569.
3939 if (NewWidth > 0 && NewWidth < Known.getBitWidth() &&
3940 shouldChangeType(Known.getBitWidth(), NewWidth)) {
3941 IntegerType *Ty = IntegerType::get(SI.getContext(), NewWidth);
3943 Value *NewCond = Builder.CreateTrunc(Cond, Ty, "trunc");
3944
3945 for (auto Case : SI.cases()) {
3946 APInt TruncatedCase = Case.getCaseValue()->getValue().trunc(NewWidth);
3947 Case.setValue(ConstantInt::get(SI.getContext(), TruncatedCase));
3948 }
3949 return replaceOperand(SI, 0, NewCond);
3950 }
3951
3952 if (isa<UndefValue>(Cond)) {
3953 handlePotentiallyDeadSuccessors(SI.getParent(), /*LiveSucc*/ nullptr);
3954 return nullptr;
3955 }
3956 if (auto *CI = dyn_cast<ConstantInt>(Cond)) {
3957 handlePotentiallyDeadSuccessors(SI.getParent(),
3958 SI.findCaseValue(CI)->getCaseSuccessor());
3959 return nullptr;
3960 }
3961
3962 return nullptr;
3963}
3964
3966InstCombinerImpl::foldExtractOfOverflowIntrinsic(ExtractValueInst &EV) {
3967 auto *WO = dyn_cast<WithOverflowInst>(EV.getAggregateOperand());
3968 if (!WO)
3969 return nullptr;
3970
3971 Intrinsic::ID OvID = WO->getIntrinsicID();
3972 const APInt *C = nullptr;
3973 if (match(WO->getRHS(), m_APIntAllowPoison(C))) {
3974 if (*EV.idx_begin() == 0 && (OvID == Intrinsic::smul_with_overflow ||
3975 OvID == Intrinsic::umul_with_overflow)) {
3976 // extractvalue (any_mul_with_overflow X, -1), 0 --> -X
3977 if (C->isAllOnes())
3978 return BinaryOperator::CreateNeg(WO->getLHS());
3979 // extractvalue (any_mul_with_overflow X, 2^n), 0 --> X << n
3980 if (C->isPowerOf2()) {
3981 return BinaryOperator::CreateShl(
3982 WO->getLHS(),
3983 ConstantInt::get(WO->getLHS()->getType(), C->logBase2()));
3984 }
3985 }
3986 }
3987
3988 // We're extracting from an overflow intrinsic. See if we're the only user.
3989 // That allows us to simplify multiple result intrinsics to simpler things
3990 // that just get one value.
3991 if (!WO->hasOneUse())
3992 return nullptr;
3993
3994 // Check if we're grabbing only the result of a 'with overflow' intrinsic
3995 // and replace it with a traditional binary instruction.
3996 if (*EV.idx_begin() == 0) {
3997 Instruction::BinaryOps BinOp = WO->getBinaryOp();
3998 Value *LHS = WO->getLHS(), *RHS = WO->getRHS();
3999 // Replace the old instruction's uses with poison.
4000 replaceInstUsesWith(*WO, PoisonValue::get(WO->getType()));
4002 return BinaryOperator::Create(BinOp, LHS, RHS);
4003 }
4004
4005 assert(*EV.idx_begin() == 1 && "Unexpected extract index for overflow inst");
4006
4007 // (usub LHS, RHS) overflows when LHS is unsigned-less-than RHS.
4008 if (OvID == Intrinsic::usub_with_overflow)
4009 return new ICmpInst(ICmpInst::ICMP_ULT, WO->getLHS(), WO->getRHS());
4010
4011 // smul with i1 types overflows when both sides are set: -1 * -1 == +1, but
4012 // +1 is not possible because we assume signed values.
4013 if (OvID == Intrinsic::smul_with_overflow &&
4014 WO->getLHS()->getType()->isIntOrIntVectorTy(1))
4015 return BinaryOperator::CreateAnd(WO->getLHS(), WO->getRHS());
4016
4017 // extractvalue (umul_with_overflow X, X), 1 -> X u> 2^(N/2)-1
4018 if (OvID == Intrinsic::umul_with_overflow && WO->getLHS() == WO->getRHS()) {
4019 unsigned BitWidth = WO->getLHS()->getType()->getScalarSizeInBits();
4020 // Only handle even bitwidths for performance reasons.
4021 if (BitWidth % 2 == 0)
4022 return new ICmpInst(
4023 ICmpInst::ICMP_UGT, WO->getLHS(),
4024 ConstantInt::get(WO->getLHS()->getType(),
4026 }
4027
4028 // If only the overflow result is used, and the right hand side is a
4029 // constant (or constant splat), we can remove the intrinsic by directly
4030 // checking for overflow.
4031 if (C) {
4032 // Compute the no-wrap range for LHS given RHS=C, then construct an
4033 // equivalent icmp, potentially using an offset.
4035 WO->getBinaryOp(), *C, WO->getNoWrapKind());
4036
4037 CmpInst::Predicate Pred;
4038 APInt NewRHSC, Offset;
4039 NWR.getEquivalentICmp(Pred, NewRHSC, Offset);
4040 auto *OpTy = WO->getRHS()->getType();
4041 auto *NewLHS = WO->getLHS();
4042 if (Offset != 0)
4043 NewLHS = Builder.CreateAdd(NewLHS, ConstantInt::get(OpTy, Offset));
4044 return new ICmpInst(ICmpInst::getInversePredicate(Pred), NewLHS,
4045 ConstantInt::get(OpTy, NewRHSC));
4046 }
4047
4048 return nullptr;
4049}
4050
4052 Value *Agg = EV.getAggregateOperand();
4053
4054 if (!EV.hasIndices())
4055 return replaceInstUsesWith(EV, Agg);
4056
4057 if (Value *V = simplifyExtractValueInst(Agg, EV.getIndices(),
4058 SQ.getWithInstruction(&EV)))
4059 return replaceInstUsesWith(EV, V);
4060
4061 if (InsertValueInst *IV = dyn_cast<InsertValueInst>(Agg)) {
4062 // We're extracting from an insertvalue instruction, compare the indices
4063 const unsigned *exti, *exte, *insi, *inse;
4064 for (exti = EV.idx_begin(), insi = IV->idx_begin(),
4065 exte = EV.idx_end(), inse = IV->idx_end();
4066 exti != exte && insi != inse;
4067 ++exti, ++insi) {
4068 if (*insi != *exti)
4069 // The insert and extract both reference distinctly different elements.
4070 // This means the extract is not influenced by the insert, and we can
4071 // replace the aggregate operand of the extract with the aggregate
4072 // operand of the insert. i.e., replace
4073 // %I = insertvalue { i32, { i32 } } %A, { i32 } { i32 42 }, 1
4074 // %E = extractvalue { i32, { i32 } } %I, 0
4075 // with
4076 // %E = extractvalue { i32, { i32 } } %A, 0
4077 return ExtractValueInst::Create(IV->getAggregateOperand(),
4078 EV.getIndices());
4079 }
4080 if (exti == exte && insi == inse)
4081 // Both iterators are at the end: Index lists are identical. Replace
4082 // %B = insertvalue { i32, { i32 } } %A, i32 42, 1, 0
4083 // %C = extractvalue { i32, { i32 } } %B, 1, 0
4084 // with "i32 42"
4085 return replaceInstUsesWith(EV, IV->getInsertedValueOperand());
4086 if (exti == exte) {
4087 // The extract list is a prefix of the insert list. i.e. replace
4088 // %I = insertvalue { i32, { i32 } } %A, i32 42, 1, 0
4089 // %E = extractvalue { i32, { i32 } } %I, 1
4090 // with
4091 // %X = extractvalue { i32, { i32 } } %A, 1
4092 // %E = insertvalue { i32 } %X, i32 42, 0
4093 // by switching the order of the insert and extract (though the
4094 // insertvalue should be left in, since it may have other uses).
4095 Value *NewEV = Builder.CreateExtractValue(IV->getAggregateOperand(),
4096 EV.getIndices());
4097 return InsertValueInst::Create(NewEV, IV->getInsertedValueOperand(),
4098 ArrayRef(insi, inse));
4099 }
4100 if (insi == inse)
4101 // The insert list is a prefix of the extract list
4102 // We can simply remove the common indices from the extract and make it
4103 // operate on the inserted value instead of the insertvalue result.
4104 // i.e., replace
4105 // %I = insertvalue { i32, { i32 } } %A, { i32 } { i32 42 }, 1
4106 // %E = extractvalue { i32, { i32 } } %I, 1, 0
4107 // with
4108 // %E extractvalue { i32 } { i32 42 }, 0
4109 return ExtractValueInst::Create(IV->getInsertedValueOperand(),
4110 ArrayRef(exti, exte));
4111 }
4112
4113 if (Instruction *R = foldExtractOfOverflowIntrinsic(EV))
4114 return R;
4115
4116 if (LoadInst *L = dyn_cast<LoadInst>(Agg)) {
4117 // Bail out if the aggregate contains scalable vector type
4118 if (auto *STy = dyn_cast<StructType>(Agg->getType());
4119 STy && STy->isScalableTy())
4120 return nullptr;
4121
4122 // If the (non-volatile) load only has one use, we can rewrite this to a
4123 // load from a GEP. This reduces the size of the load. If a load is used
4124 // only by extractvalue instructions then this either must have been
4125 // optimized before, or it is a struct with padding, in which case we
4126 // don't want to do the transformation as it loses padding knowledge.
4127 if (L->isSimple() && L->hasOneUse()) {
4128 // extractvalue has integer indices, getelementptr has Value*s. Convert.
4129 SmallVector<Value*, 4> Indices;
4130 // Prefix an i32 0 since we need the first element.
4131 Indices.push_back(Builder.getInt32(0));
4132 for (unsigned Idx : EV.indices())
4133 Indices.push_back(Builder.getInt32(Idx));
4134
4135 // We need to insert these at the location of the old load, not at that of
4136 // the extractvalue.
4138 Value *GEP = Builder.CreateInBoundsGEP(L->getType(),
4139 L->getPointerOperand(), Indices);
4141 // Whatever aliasing information we had for the orignal load must also
4142 // hold for the smaller load, so propagate the annotations.
4143 NL->setAAMetadata(L->getAAMetadata());
4144 // Returning the load directly will cause the main loop to insert it in
4145 // the wrong spot, so use replaceInstUsesWith().
4146 return replaceInstUsesWith(EV, NL);
4147 }
4148 }
4149
4150 if (auto *PN = dyn_cast<PHINode>(Agg))
4151 if (Instruction *Res = foldOpIntoPhi(EV, PN))
4152 return Res;
4153
4154 // Canonicalize extract (select Cond, TV, FV)
4155 // -> select cond, (extract TV), (extract FV)
4156 if (auto *SI = dyn_cast<SelectInst>(Agg))
4157 if (Instruction *R = FoldOpIntoSelect(EV, SI, /*FoldWithMultiUse=*/true))
4158 return R;
4159
4160 // We could simplify extracts from other values. Note that nested extracts may
4161 // already be simplified implicitly by the above: extract (extract (insert) )
4162 // will be translated into extract ( insert ( extract ) ) first and then just
4163 // the value inserted, if appropriate. Similarly for extracts from single-use
4164 // loads: extract (extract (load)) will be translated to extract (load (gep))
4165 // and if again single-use then via load (gep (gep)) to load (gep).
4166 // However, double extracts from e.g. function arguments or return values
4167 // aren't handled yet.
4168 return nullptr;
4169}
4170
4171/// Return 'true' if the given typeinfo will match anything.
4172static bool isCatchAll(EHPersonality Personality, Constant *TypeInfo) {
4173 switch (Personality) {
4177 // The GCC C EH and Rust personality only exists to support cleanups, so
4178 // it's not clear what the semantics of catch clauses are.
4179 return false;
4181 return false;
4183 // While __gnat_all_others_value will match any Ada exception, it doesn't
4184 // match foreign exceptions (or didn't, before gcc-4.7).
4185 return false;
4196 return TypeInfo->isNullValue();
4197 }
4198 llvm_unreachable("invalid enum");
4199}
4200
4201static bool shorter_filter(const Value *LHS, const Value *RHS) {
4202 return
4203 cast<ArrayType>(LHS->getType())->getNumElements()
4204 <
4205 cast<ArrayType>(RHS->getType())->getNumElements();
4206}
4207
4209 // The logic here should be correct for any real-world personality function.
4210 // However if that turns out not to be true, the offending logic can always
4211 // be conditioned on the personality function, like the catch-all logic is.
4212 EHPersonality Personality =
4213 classifyEHPersonality(LI.getParent()->getParent()->getPersonalityFn());
4214
4215 // Simplify the list of clauses, eg by removing repeated catch clauses
4216 // (these are often created by inlining).
4217 bool MakeNewInstruction = false; // If true, recreate using the following:
4218 SmallVector<Constant *, 16> NewClauses; // - Clauses for the new instruction;
4219 bool CleanupFlag = LI.isCleanup(); // - The new instruction is a cleanup.
4220
4221 SmallPtrSet<Value *, 16> AlreadyCaught; // Typeinfos known caught already.
4222 for (unsigned i = 0, e = LI.getNumClauses(); i != e; ++i) {
4223 bool isLastClause = i + 1 == e;
4224 if (LI.isCatch(i)) {
4225 // A catch clause.
4226 Constant *CatchClause = LI.getClause(i);
4227 Constant *TypeInfo = CatchClause->stripPointerCasts();
4228
4229 // If we already saw this clause, there is no point in having a second
4230 // copy of it.
4231 if (AlreadyCaught.insert(TypeInfo).second) {
4232 // This catch clause was not already seen.
4233 NewClauses.push_back(CatchClause);
4234 } else {
4235 // Repeated catch clause - drop the redundant copy.
4236 MakeNewInstruction = true;
4237 }
4238
4239 // If this is a catch-all then there is no point in keeping any following
4240 // clauses or marking the landingpad as having a cleanup.
4241 if (isCatchAll(Personality, TypeInfo)) {
4242 if (!isLastClause)
4243 MakeNewInstruction = true;
4244 CleanupFlag = false;
4245 break;
4246 }
4247 } else {
4248 // A filter clause. If any of the filter elements were already caught
4249 // then they can be dropped from the filter. It is tempting to try to
4250 // exploit the filter further by saying that any typeinfo that does not
4251 // occur in the filter can't be caught later (and thus can be dropped).
4252 // However this would be wrong, since typeinfos can match without being
4253 // equal (for example if one represents a C++ class, and the other some
4254 // class derived from it).
4255 assert(LI.isFilter(i) && "Unsupported landingpad clause!");
4256 Constant *FilterClause = LI.getClause(i);
4257 ArrayType *FilterType = cast<ArrayType>(FilterClause->getType());
4258 unsigned NumTypeInfos = FilterType->getNumElements();
4259
4260 // An empty filter catches everything, so there is no point in keeping any
4261 // following clauses or marking the landingpad as having a cleanup. By
4262 // dealing with this case here the following code is made a bit simpler.
4263 if (!NumTypeInfos) {
4264 NewClauses.push_back(FilterClause);
4265 if (!isLastClause)
4266 MakeNewInstruction = true;
4267 CleanupFlag = false;
4268 break;
4269 }
4270
4271 bool MakeNewFilter = false; // If true, make a new filter.
4272 SmallVector<Constant *, 16> NewFilterElts; // New elements.
4273 if (isa<ConstantAggregateZero>(FilterClause)) {
4274 // Not an empty filter - it contains at least one null typeinfo.
4275 assert(NumTypeInfos > 0 && "Should have handled empty filter already!");
4276 Constant *TypeInfo =
4278 // If this typeinfo is a catch-all then the filter can never match.
4279 if (isCatchAll(Personality, TypeInfo)) {
4280 // Throw the filter away.
4281 MakeNewInstruction = true;
4282 continue;
4283 }
4284
4285 // There is no point in having multiple copies of this typeinfo, so
4286 // discard all but the first copy if there is more than one.
4287 NewFilterElts.push_back(TypeInfo);
4288 if (NumTypeInfos > 1)
4289 MakeNewFilter = true;
4290 } else {
4291 ConstantArray *Filter = cast<ConstantArray>(FilterClause);
4292 SmallPtrSet<Value *, 16> SeenInFilter; // For uniquing the elements.
4293 NewFilterElts.reserve(NumTypeInfos);
4294
4295 // Remove any filter elements that were already caught or that already
4296 // occurred in the filter. While there, see if any of the elements are
4297 // catch-alls. If so, the filter can be discarded.
4298 bool SawCatchAll = false;
4299 for (unsigned j = 0; j != NumTypeInfos; ++j) {
4300 Constant *Elt = Filter->getOperand(j);
4301 Constant *TypeInfo = Elt->stripPointerCasts();
4302 if (isCatchAll(Personality, TypeInfo)) {
4303 // This element is a catch-all. Bail out, noting this fact.
4304 SawCatchAll = true;
4305 break;
4306 }
4307
4308 // Even if we've seen a type in a catch clause, we don't want to
4309 // remove it from the filter. An unexpected type handler may be
4310 // set up for a call site which throws an exception of the same
4311 // type caught. In order for the exception thrown by the unexpected
4312 // handler to propagate correctly, the filter must be correctly
4313 // described for the call site.
4314 //
4315 // Example:
4316 //
4317 // void unexpected() { throw 1;}
4318 // void foo() throw (int) {
4319 // std::set_unexpected(unexpected);
4320 // try {
4321 // throw 2.0;
4322 // } catch (int i) {}
4323 // }
4324
4325 // There is no point in having multiple copies of the same typeinfo in
4326 // a filter, so only add it if we didn't already.
4327 if (SeenInFilter.insert(TypeInfo).second)
4328 NewFilterElts.push_back(cast<Constant>(Elt));
4329 }
4330 // A filter containing a catch-all cannot match anything by definition.
4331 if (SawCatchAll) {
4332 // Throw the filter away.
4333 MakeNewInstruction = true;
4334 continue;
4335 }
4336
4337 // If we dropped something from the filter, make a new one.
4338 if (NewFilterElts.size() < NumTypeInfos)
4339 MakeNewFilter = true;
4340 }
4341 if (MakeNewFilter) {
4342 FilterType = ArrayType::get(FilterType->getElementType(),
4343 NewFilterElts.size());
4344 FilterClause = ConstantArray::get(FilterType, NewFilterElts);
4345 MakeNewInstruction = true;
4346 }
4347
4348 NewClauses.push_back(FilterClause);
4349
4350 // If the new filter is empty then it will catch everything so there is
4351 // no point in keeping any following clauses or marking the landingpad
4352 // as having a cleanup. The case of the original filter being empty was
4353 // already handled above.
4354 if (MakeNewFilter && !NewFilterElts.size()) {
4355 assert(MakeNewInstruction && "New filter but not a new instruction!");
4356 CleanupFlag = false;
4357 break;
4358 }
4359 }
4360 }
4361
4362 // If several filters occur in a row then reorder them so that the shortest
4363 // filters come first (those with the smallest number of elements). This is
4364 // advantageous because shorter filters are more likely to match, speeding up
4365 // unwinding, but mostly because it increases the effectiveness of the other
4366 // filter optimizations below.
4367 for (unsigned i = 0, e = NewClauses.size(); i + 1 < e; ) {
4368 unsigned j;
4369 // Find the maximal 'j' s.t. the range [i, j) consists entirely of filters.
4370 for (j = i; j != e; ++j)
4371 if (!isa<ArrayType>(NewClauses[j]->getType()))
4372 break;
4373
4374 // Check whether the filters are already sorted by length. We need to know
4375 // if sorting them is actually going to do anything so that we only make a
4376 // new landingpad instruction if it does.
4377 for (unsigned k = i; k + 1 < j; ++k)
4378 if (shorter_filter(NewClauses[k+1], NewClauses[k])) {
4379 // Not sorted, so sort the filters now. Doing an unstable sort would be
4380 // correct too but reordering filters pointlessly might confuse users.
4381 std::stable_sort(NewClauses.begin() + i, NewClauses.begin() + j,
4383 MakeNewInstruction = true;
4384 break;
4385 }
4386
4387 // Look for the next batch of filters.
4388 i = j + 1;
4389 }
4390
4391 // If typeinfos matched if and only if equal, then the elements of a filter L
4392 // that occurs later than a filter F could be replaced by the intersection of
4393 // the elements of F and L. In reality two typeinfos can match without being
4394 // equal (for example if one represents a C++ class, and the other some class
4395 // derived from it) so it would be wrong to perform this transform in general.
4396 // However the transform is correct and useful if F is a subset of L. In that
4397 // case L can be replaced by F, and thus removed altogether since repeating a
4398 // filter is pointless. So here we look at all pairs of filters F and L where
4399 // L follows F in the list of clauses, and remove L if every element of F is
4400 // an element of L. This can occur when inlining C++ functions with exception
4401 // specifications.
4402 for (unsigned i = 0; i + 1 < NewClauses.size(); ++i) {
4403 // Examine each filter in turn.
4404 Value *Filter = NewClauses[i];
4405 ArrayType *FTy = dyn_cast<ArrayType>(Filter->getType());
4406 if (!FTy)
4407 // Not a filter - skip it.
4408 continue;
4409 unsigned FElts = FTy->getNumElements();
4410 // Examine each filter following this one. Doing this backwards means that
4411 // we don't have to worry about filters disappearing under us when removed.
4412 for (unsigned j = NewClauses.size() - 1; j != i; --j) {
4413 Value *LFilter = NewClauses[j];
4414 ArrayType *LTy = dyn_cast<ArrayType>(LFilter->getType());
4415 if (!LTy)
4416 // Not a filter - skip it.
4417 continue;
4418 // If Filter is a subset of LFilter, i.e. every element of Filter is also
4419 // an element of LFilter, then discard LFilter.
4420 SmallVectorImpl<Constant *>::iterator J = NewClauses.begin() + j;
4421 // If Filter is empty then it is a subset of LFilter.
4422 if (!FElts) {
4423 // Discard LFilter.
4424 NewClauses.erase(J);
4425 MakeNewInstruction = true;
4426 // Move on to the next filter.
4427 continue;
4428 }
4429 unsigned LElts = LTy->getNumElements();
4430 // If Filter is longer than LFilter then it cannot be a subset of it.
4431 if (FElts > LElts)
4432 // Move on to the next filter.
4433 continue;
4434 // At this point we know that LFilter has at least one element.
4435 if (isa<ConstantAggregateZero>(LFilter)) { // LFilter only contains zeros.
4436 // Filter is a subset of LFilter iff Filter contains only zeros (as we
4437 // already know that Filter is not longer than LFilter).
4438 if (isa<ConstantAggregateZero>(Filter)) {
4439 assert(FElts <= LElts && "Should have handled this case earlier!");
4440 // Discard LFilter.
4441 NewClauses.erase(J);
4442 MakeNewInstruction = true;
4443 }
4444 // Move on to the next filter.
4445 continue;
4446 }
4447 ConstantArray *LArray = cast<ConstantArray>(LFilter);
4448 if (isa<ConstantAggregateZero>(Filter)) { // Filter only contains zeros.
4449 // Since Filter is non-empty and contains only zeros, it is a subset of
4450 // LFilter iff LFilter contains a zero.
4451 assert(FElts > 0 && "Should have eliminated the empty filter earlier!");
4452 for (unsigned l = 0; l != LElts; ++l)
4453 if (LArray->getOperand(l)->isNullValue()) {
4454 // LFilter contains a zero - discard it.
4455 NewClauses.erase(J);
4456 MakeNewInstruction = true;
4457 break;
4458 }
4459 // Move on to the next filter.
4460 continue;
4461 }
4462 // At this point we know that both filters are ConstantArrays. Loop over
4463 // operands to see whether every element of Filter is also an element of
4464 // LFilter. Since filters tend to be short this is probably faster than
4465 // using a method that scales nicely.
4466 ConstantArray *FArray = cast<ConstantArray>(Filter);
4467 bool AllFound = true;
4468 for (unsigned f = 0; f != FElts; ++f) {
4469 Value *FTypeInfo = FArray->getOperand(f)->stripPointerCasts();
4470 AllFound = false;
4471 for (unsigned l = 0; l != LElts; ++l) {
4472 Value *LTypeInfo = LArray->getOperand(l)->stripPointerCasts();
4473 if (LTypeInfo == FTypeInfo) {
4474 AllFound = true;
4475 break;
4476 }
4477 }
4478 if (!AllFound)
4479 break;
4480 }
4481 if (AllFound) {
4482 // Discard LFilter.
4483 NewClauses.erase(J);
4484 MakeNewInstruction = true;
4485 }
4486 // Move on to the next filter.
4487 }
4488 }
4489
4490 // If we changed any of the clauses, replace the old landingpad instruction
4491 // with a new one.
4492 if (MakeNewInstruction) {
4494 NewClauses.size());
4495 for (Constant *C : NewClauses)
4496 NLI->addClause(C);
4497 // A landing pad with no clauses must have the cleanup flag set. It is
4498 // theoretically possible, though highly unlikely, that we eliminated all
4499 // clauses. If so, force the cleanup flag to true.
4500 if (NewClauses.empty())
4501 CleanupFlag = true;
4502 NLI->setCleanup(CleanupFlag);
4503 return NLI;
4504 }
4505
4506 // Even if none of the clauses changed, we may nonetheless have understood
4507 // that the cleanup flag is pointless. Clear it if so.
4508 if (LI.isCleanup() != CleanupFlag) {
4509 assert(!CleanupFlag && "Adding a cleanup, not removing one?!");
4510 LI.setCleanup(CleanupFlag);
4511 return &LI;
4512 }
4513
4514 return nullptr;
4515}
4516
4517Value *
4519 // Try to push freeze through instructions that propagate but don't produce
4520 // poison as far as possible. If an operand of freeze follows three
4521 // conditions 1) one-use, 2) does not produce poison, and 3) has all but one
4522 // guaranteed-non-poison operands then push the freeze through to the one
4523 // operand that is not guaranteed non-poison. The actual transform is as
4524 // follows.
4525 // Op1 = ... ; Op1 can be posion
4526 // Op0 = Inst(Op1, NonPoisonOps...) ; Op0 has only one use and only have
4527 // ; single guaranteed-non-poison operands
4528 // ... = Freeze(Op0)
4529 // =>
4530 // Op1 = ...
4531 // Op1.fr = Freeze(Op1)
4532 // ... = Inst(Op1.fr, NonPoisonOps...)
4533 auto *OrigOp = OrigFI.getOperand(0);
4534 auto *OrigOpInst = dyn_cast<Instruction>(OrigOp);
4535
4536 // While we could change the other users of OrigOp to use freeze(OrigOp), that
4537 // potentially reduces their optimization potential, so let's only do this iff
4538 // the OrigOp is only used by the freeze.
4539 if (!OrigOpInst || !OrigOpInst->hasOneUse() || isa<PHINode>(OrigOp))
4540 return nullptr;
4541
4542 // We can't push the freeze through an instruction which can itself create
4543 // poison. If the only source of new poison is flags, we can simply
4544 // strip them (since we know the only use is the freeze and nothing can
4545 // benefit from them.)
4546 if (canCreateUndefOrPoison(cast<Operator>(OrigOp),
4547 /*ConsiderFlagsAndMetadata*/ false))
4548 return nullptr;
4549
4550 // If operand is guaranteed not to be poison, there is no need to add freeze
4551 // to the operand. So we first find the operand that is not guaranteed to be
4552 // poison.
4553 Use *MaybePoisonOperand = nullptr;
4554 for (Use &U : OrigOpInst->operands()) {
4555 if (isa<MetadataAsValue>(U.get()) ||
4557 continue;
4558 if (!MaybePoisonOperand)
4559 MaybePoisonOperand = &U;
4560 else
4561 return nullptr;
4562 }
4563
4564 OrigOpInst->dropPoisonGeneratingAnnotations();
4565
4566 // If all operands are guaranteed to be non-poison, we can drop freeze.
4567 if (!MaybePoisonOperand)
4568 return OrigOp;
4569
4570 Builder.SetInsertPoint(OrigOpInst);
4571 auto *FrozenMaybePoisonOperand = Builder.CreateFreeze(
4572 MaybePoisonOperand->get(), MaybePoisonOperand->get()->getName() + ".fr");
4573
4574 replaceUse(*MaybePoisonOperand, FrozenMaybePoisonOperand);
4575 return OrigOp;
4576}
4577
4579 PHINode *PN) {
4580 // Detect whether this is a recurrence with a start value and some number of
4581 // backedge values. We'll check whether we can push the freeze through the
4582 // backedge values (possibly dropping poison flags along the way) until we
4583 // reach the phi again. In that case, we can move the freeze to the start
4584 // value.
4585 Use *StartU = nullptr;
4587 for (Use &U : PN->incoming_values()) {
4588 if (DT.dominates(PN->getParent(), PN->getIncomingBlock(U))) {
4589 // Add backedge value to worklist.
4590 Worklist.push_back(U.get());
4591 continue;
4592 }
4593
4594 // Don't bother handling multiple start values.
4595 if (StartU)
4596 return nullptr;
4597 StartU = &U;
4598 }
4599
4600 if (!StartU || Worklist.empty())
4601 return nullptr; // Not a recurrence.
4602
4603 Value *StartV = StartU->get();
4604 BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
4605 bool StartNeedsFreeze = !isGuaranteedNotToBeUndefOrPoison(StartV);
4606 // We can't insert freeze if the start value is the result of the
4607 // terminator (e.g. an invoke).
4608 if (StartNeedsFreeze && StartBB->getTerminator() == StartV)
4609 return nullptr;
4610
4613 while (!Worklist.empty()) {
4614 Value *V = Worklist.pop_back_val();
4615 if (!Visited.insert(V).second)
4616 continue;
4617
4618 if (Visited.size() > 32)
4619 return nullptr; // Limit the total number of values we inspect.
4620
4621 // Assume that PN is non-poison, because it will be after the transform.
4622 if (V == PN || isGuaranteedNotToBeUndefOrPoison(V))
4623 continue;
4624
4625 Instruction *I = dyn_cast<Instruction>(V);
4626 if (!I || canCreateUndefOrPoison(cast<Operator>(I),
4627 /*ConsiderFlagsAndMetadata*/ false))
4628 return nullptr;
4629
4630 DropFlags.push_back(I);
4631 append_range(Worklist, I->operands());
4632 }
4633
4634 for (Instruction *I : DropFlags)
4635 I->dropPoisonGeneratingAnnotations();
4636
4637 if (StartNeedsFreeze) {
4639 Value *FrozenStartV = Builder.CreateFreeze(StartV,
4640 StartV->getName() + ".fr");
4641 replaceUse(*StartU, FrozenStartV);
4642 }
4643 return replaceInstUsesWith(FI, PN);
4644}
4645
4647 Value *Op = FI.getOperand(0);
4648
4649 if (isa<Constant>(Op) || Op->hasOneUse())
4650 return false;
4651
4652 // Move the freeze directly after the definition of its operand, so that
4653 // it dominates the maximum number of uses. Note that it may not dominate
4654 // *all* uses if the operand is an invoke/callbr and the use is in a phi on
4655 // the normal/default destination. This is why the domination check in the
4656 // replacement below is still necessary.
4657 BasicBlock::iterator MoveBefore;
4658 if (isa<Argument>(Op)) {
4659 MoveBefore =
4661 } else {
4662 auto MoveBeforeOpt = cast<Instruction>(Op)->getInsertionPointAfterDef();
4663 if (!MoveBeforeOpt)
4664 return false;
4665 MoveBefore = *MoveBeforeOpt;
4666 }
4667
4668 // Don't move to the position of a debug intrinsic.
4669 if (isa<DbgInfoIntrinsic>(MoveBefore))
4670 MoveBefore = MoveBefore->getNextNonDebugInstruction()->getIterator();
4671 // Re-point iterator to come after any debug-info records, if we're
4672 // running in "RemoveDIs" mode
4673 MoveBefore.setHeadBit(false);
4674
4675 bool Changed = false;
4676 if (&FI != &*MoveBefore) {
4677 FI.moveBefore(*MoveBefore->getParent(), MoveBefore);
4678 Changed = true;
4679 }
4680
4681 Op->replaceUsesWithIf(&FI, [&](Use &U) -> bool {
4682 bool Dominates = DT.dominates(&FI, U);
4683 Changed |= Dominates;
4684 return Dominates;
4685 });
4686
4687 return Changed;
4688}
4689
4690// Check if any direct or bitcast user of this value is a shuffle instruction.
4692 for (auto *U : V->users()) {
4693 if (isa<ShuffleVectorInst>(U))
4694 return true;
4695 else if (match(U, m_BitCast(m_Specific(V))) && isUsedWithinShuffleVector(U))
4696 return true;
4697 }
4698 return false;
4699}
4700
4702 Value *Op0 = I.getOperand(0);
4703
4705 return replaceInstUsesWith(I, V);
4706
4707 // freeze (phi const, x) --> phi const, (freeze x)
4708 if (auto *PN = dyn_cast<PHINode>(Op0)) {
4709 if (Instruction *NV = foldOpIntoPhi(I, PN))
4710 return NV;
4711 if (Instruction *NV = foldFreezeIntoRecurrence(I, PN))
4712 return NV;
4713 }
4714
4716 return replaceInstUsesWith(I, NI);
4717
4718 // If I is freeze(undef), check its uses and fold it to a fixed constant.
4719 // - or: pick -1
4720 // - select's condition: if the true value is constant, choose it by making
4721 // the condition true.
4722 // - default: pick 0
4723 //
4724 // Note that this transform is intentionally done here rather than
4725 // via an analysis in InstSimplify or at individual user sites. That is
4726 // because we must produce the same value for all uses of the freeze -
4727 // it's the reason "freeze" exists!
4728 //
4729 // TODO: This could use getBinopAbsorber() / getBinopIdentity() to avoid
4730 // duplicating logic for binops at least.
4731 auto getUndefReplacement = [&I](Type *Ty) {
4732 Constant *BestValue = nullptr;
4733 Constant *NullValue = Constant::getNullValue(Ty);
4734 for (const auto *U : I.users()) {
4735 Constant *C = NullValue;
4736 if (match(U, m_Or(m_Value(), m_Value())))
4738 else if (match(U, m_Select(m_Specific(&I), m_Constant(), m_Value())))
4739 C = ConstantInt::getTrue(Ty);
4740
4741 if (!BestValue)
4742 BestValue = C;
4743 else if (BestValue != C)
4744 BestValue = NullValue;
4745 }
4746 assert(BestValue && "Must have at least one use");
4747 return BestValue;
4748 };
4749
4750 if (match(Op0, m_Undef())) {
4751 // Don't fold freeze(undef/poison) if it's used as a vector operand in
4752 // a shuffle. This may improve codegen for shuffles that allow
4753 // unspecified inputs.
4755 return nullptr;
4756 return replaceInstUsesWith(I, getUndefReplacement(I.getType()));
4757 }
4758
4759 Constant *C;
4760 if (match(Op0, m_Constant(C)) && C->containsUndefOrPoisonElement()) {
4761 Constant *ReplaceC = getUndefReplacement(I.getType()->getScalarType());
4763 }
4764
4765 // Replace uses of Op with freeze(Op).
4766 if (freezeOtherUses(I))
4767 return &I;
4768
4769 return nullptr;
4770}
4771
4772/// Check for case where the call writes to an otherwise dead alloca. This
4773/// shows up for unused out-params in idiomatic C/C++ code. Note that this
4774/// helper *only* analyzes the write; doesn't check any other legality aspect.
4776 auto *CB = dyn_cast<CallBase>(I);
4777 if (!CB)
4778 // TODO: handle e.g. store to alloca here - only worth doing if we extend
4779 // to allow reload along used path as described below. Otherwise, this
4780 // is simply a store to a dead allocation which will be removed.
4781 return false;
4782 std::optional<MemoryLocation> Dest = MemoryLocation::getForDest(CB, TLI);
4783 if (!Dest)
4784 return false;
4785 auto *AI = dyn_cast<AllocaInst>(getUnderlyingObject(Dest->Ptr));
4786 if (!AI)
4787 // TODO: allow malloc?
4788 return false;
4789 // TODO: allow memory access dominated by move point? Note that since AI
4790 // could have a reference to itself captured by the call, we would need to
4791 // account for cycles in doing so.
4792 SmallVector<const User *> AllocaUsers;
4794 auto pushUsers = [&](const Instruction &I) {
4795 for (const User *U : I.users()) {
4796 if (Visited.insert(U).second)
4797 AllocaUsers.push_back(U);
4798 }
4799 };
4800 pushUsers(*AI);
4801 while (!AllocaUsers.empty()) {
4802 auto *UserI = cast<Instruction>(AllocaUsers.pop_back_val());
4803 if (isa<GetElementPtrInst>(UserI) || isa<AddrSpaceCastInst>(UserI)) {
4804 pushUsers(*UserI);
4805 continue;
4806 }
4807 if (UserI == CB)
4808 continue;
4809 // TODO: support lifetime.start/end here
4810 return false;
4811 }
4812 return true;
4813}
4814
4815/// Try to move the specified instruction from its current block into the
4816/// beginning of DestBlock, which can only happen if it's safe to move the
4817/// instruction past all of the instructions between it and the end of its
4818/// block.
4820 BasicBlock *DestBlock) {
4821 BasicBlock *SrcBlock = I->getParent();
4822
4823 // Cannot move control-flow-involving, volatile loads, vaarg, etc.
4824 if (isa<PHINode>(I) || I->isEHPad() || I->mayThrow() || !I->willReturn() ||
4825 I->isTerminator())
4826 return false;
4827
4828 // Do not sink static or dynamic alloca instructions. Static allocas must
4829 // remain in the entry block, and dynamic allocas must not be sunk in between
4830 // a stacksave / stackrestore pair, which would incorrectly shorten its
4831 // lifetime.
4832 if (isa<AllocaInst>(I))
4833 return false;
4834
4835 // Do not sink into catchswitch blocks.
4836 if (isa<CatchSwitchInst>(DestBlock->getTerminator()))
4837 return false;
4838
4839 // Do not sink convergent call instructions.
4840 if (auto *CI = dyn_cast<CallInst>(I)) {
4841 if (CI->isConvergent())
4842 return false;
4843 }
4844
4845 // Unless we can prove that the memory write isn't visibile except on the
4846 // path we're sinking to, we must bail.
4847 if (I->mayWriteToMemory()) {
4848 if (!SoleWriteToDeadLocal(I, TLI))
4849 return false;
4850 }
4851
4852 // We can only sink load instructions if there is nothing between the load and
4853 // the end of block that could change the value.
4854 if (I->mayReadFromMemory() &&
4855 !I->hasMetadata(LLVMContext::MD_invariant_load)) {
4856 // We don't want to do any sophisticated alias analysis, so we only check
4857 // the instructions after I in I's parent block if we try to sink to its
4858 // successor block.
4859 if (DestBlock->getUniquePredecessor() != I->getParent())
4860 return false;
4861 for (BasicBlock::iterator Scan = std::next(I->getIterator()),
4862 E = I->getParent()->end();
4863 Scan != E; ++Scan)
4864 if (Scan->mayWriteToMemory())
4865 return false;
4866 }
4867
4868 I->dropDroppableUses([&](const Use *U) {
4869 auto *I = dyn_cast<Instruction>(U->getUser());
4870 if (I && I->getParent() != DestBlock) {
4871 Worklist.add(I);
4872 return true;
4873 }
4874 return false;
4875 });
4876 /// FIXME: We could remove droppable uses that are not dominated by
4877 /// the new position.
4878
4879 BasicBlock::iterator InsertPos = DestBlock->getFirstInsertionPt();
4880 I->moveBefore(*DestBlock, InsertPos);
4881 ++NumSunkInst;
4882
4883 // Also sink all related debug uses from the source basic block. Otherwise we
4884 // get debug use before the def. Attempt to salvage debug uses first, to
4885 // maximise the range variables have location for. If we cannot salvage, then
4886 // mark the location undef: we know it was supposed to receive a new location
4887 // here, but that computation has been sunk.
4889 SmallVector<DbgVariableRecord *, 2> DbgVariableRecords;
4890 findDbgUsers(DbgUsers, I, &DbgVariableRecords);
4891 if (!DbgUsers.empty())
4892 tryToSinkInstructionDbgValues(I, InsertPos, SrcBlock, DestBlock, DbgUsers);
4893 if (!DbgVariableRecords.empty())
4894 tryToSinkInstructionDbgVariableRecords(I, InsertPos, SrcBlock, DestBlock,
4895 DbgVariableRecords);
4896
4897 // PS: there are numerous flaws with this behaviour, not least that right now
4898 // assignments can be re-ordered past other assignments to the same variable
4899 // if they use different Values. Creating more undef assignements can never be
4900 // undone. And salvaging all users outside of this block can un-necessarily
4901 // alter the lifetime of the live-value that the variable refers to.
4902 // Some of these things can be resolved by tolerating debug use-before-defs in
4903 // LLVM-IR, however it depends on the instruction-referencing CodeGen backend
4904 // being used for more architectures.
4905
4906 return true;
4907}
4908
4910 Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
4912 // For all debug values in the destination block, the sunk instruction
4913 // will still be available, so they do not need to be dropped.
4915 for (auto &DbgUser : DbgUsers)
4916 if (DbgUser->getParent() != DestBlock)
4917 DbgUsersToSalvage.push_back(DbgUser);
4918
4919 // Process the sinking DbgUsersToSalvage in reverse order, as we only want
4920 // to clone the last appearing debug intrinsic for each given variable.
4922 for (DbgVariableIntrinsic *DVI : DbgUsersToSalvage)
4923 if (DVI->getParent() == SrcBlock)
4924 DbgUsersToSink.push_back(DVI);
4925 llvm::sort(DbgUsersToSink,
4926 [](auto *A, auto *B) { return B->comesBefore(A); });
4927
4929 SmallSet<DebugVariable, 4> SunkVariables;
4930 for (auto *User : DbgUsersToSink) {
4931 // A dbg.declare instruction should not be cloned, since there can only be
4932 // one per variable fragment. It should be left in the original place
4933 // because the sunk instruction is not an alloca (otherwise we could not be
4934 // here).
4935 if (isa<DbgDeclareInst>(User))
4936 continue;
4937
4938 DebugVariable DbgUserVariable =
4939 DebugVariable(User->getVariable(), User->getExpression(),
4940 User->getDebugLoc()->getInlinedAt());
4941
4942 if (!SunkVariables.insert(DbgUserVariable).second)
4943 continue;
4944
4945 // Leave dbg.assign intrinsics in their original positions and there should
4946 // be no need to insert a clone.
4947 if (isa<DbgAssignIntrinsic>(User))
4948 continue;
4949
4950 DIIClones.emplace_back(cast<DbgVariableIntrinsic>(User->clone()));
4951 if (isa<DbgDeclareInst>(User) && isa<CastInst>(I))
4952 DIIClones.back()->replaceVariableLocationOp(I, I->getOperand(0));
4953 LLVM_DEBUG(dbgs() << "CLONE: " << *DIIClones.back() << '\n');
4954 }
4955
4956 // Perform salvaging without the clones, then sink the clones.
4957 if (!DIIClones.empty()) {
4958 salvageDebugInfoForDbgValues(*I, DbgUsersToSalvage, {});
4959 // The clones are in reverse order of original appearance, reverse again to
4960 // maintain the original order.
4961 for (auto &DIIClone : llvm::reverse(DIIClones)) {
4962 DIIClone->insertBefore(&*InsertPos);
4963 LLVM_DEBUG(dbgs() << "SINK: " << *DIIClone << '\n');
4964 }
4965 }
4966}
4967
4969 Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
4970 BasicBlock *DestBlock,
4971 SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords) {
4972 // Implementation of tryToSinkInstructionDbgValues, but for the
4973 // DbgVariableRecord of variable assignments rather than dbg.values.
4974
4975 // Fetch all DbgVariableRecords not already in the destination.
4976 SmallVector<DbgVariableRecord *, 2> DbgVariableRecordsToSalvage;
4977 for (auto &DVR : DbgVariableRecords)
4978 if (DVR->getParent() != DestBlock)
4979 DbgVariableRecordsToSalvage.push_back(DVR);
4980
4981 // Fetch a second collection, of DbgVariableRecords in the source block that
4982 // we're going to sink.
4983 SmallVector<DbgVariableRecord *> DbgVariableRecordsToSink;
4984 for (DbgVariableRecord *DVR : DbgVariableRecordsToSalvage)
4985 if (DVR->getParent() == SrcBlock)
4986 DbgVariableRecordsToSink.push_back(DVR);
4987
4988 // Sort DbgVariableRecords according to their position in the block. This is a
4989 // partial order: DbgVariableRecords attached to different instructions will
4990 // be ordered by the instruction order, but DbgVariableRecords attached to the
4991 // same instruction won't have an order.
4992 auto Order = [](DbgVariableRecord *A, DbgVariableRecord *B) -> bool {
4993 return B->getInstruction()->comesBefore(A->getInstruction());
4994 };
4995 llvm::stable_sort(DbgVariableRecordsToSink, Order);
4996
4997 // If there are two assignments to the same variable attached to the same
4998 // instruction, the ordering between the two assignments is important. Scan
4999 // for this (rare) case and establish which is the last assignment.
5000 using InstVarPair = std::pair<const Instruction *, DebugVariable>;
5002 if (DbgVariableRecordsToSink.size() > 1) {
5004 // Count how many assignments to each variable there is per instruction.
5005 for (DbgVariableRecord *DVR : DbgVariableRecordsToSink) {
5006 DebugVariable DbgUserVariable =
5007 DebugVariable(DVR->getVariable(), DVR->getExpression(),
5008 DVR->getDebugLoc()->getInlinedAt());
5009 CountMap[std::make_pair(DVR->getInstruction(), DbgUserVariable)] += 1;
5010 }
5011
5012 // If there are any instructions with two assignments, add them to the
5013 // FilterOutMap to record that they need extra filtering.
5015 for (auto It : CountMap) {
5016 if (It.second > 1) {
5017 FilterOutMap[It.first] = nullptr;
5018 DupSet.insert(It.first.first);
5019 }
5020 }
5021
5022 // For all instruction/variable pairs needing extra filtering, find the
5023 // latest assignment.
5024 for (const Instruction *Inst : DupSet) {
5025 for (DbgVariableRecord &DVR :
5026 llvm::reverse(filterDbgVars(Inst->getDbgRecordRange()))) {
5027 DebugVariable DbgUserVariable =
5028 DebugVariable(DVR.getVariable(), DVR.getExpression(),
5029 DVR.getDebugLoc()->getInlinedAt());
5030 auto FilterIt =
5031 FilterOutMap.find(std::make_pair(Inst, DbgUserVariable));
5032 if (FilterIt == FilterOutMap.end())
5033 continue;
5034 if (FilterIt->second != nullptr)
5035 continue;
5036 FilterIt->second = &DVR;
5037 }
5038 }
5039 }
5040
5041 // Perform cloning of the DbgVariableRecords that we plan on sinking, filter
5042 // out any duplicate assignments identified above.
5044 SmallSet<DebugVariable, 4> SunkVariables;
5045 for (DbgVariableRecord *DVR : DbgVariableRecordsToSink) {
5047 continue;
5048
5049 DebugVariable DbgUserVariable =
5050 DebugVariable(DVR->getVariable(), DVR->getExpression(),
5051 DVR->getDebugLoc()->getInlinedAt());
5052
5053 // For any variable where there were multiple assignments in the same place,
5054 // ignore all but the last assignment.
5055 if (!FilterOutMap.empty()) {
5056 InstVarPair IVP = std::make_pair(DVR->getInstruction(), DbgUserVariable);
5057 auto It = FilterOutMap.find(IVP);
5058
5059 // Filter out.
5060 if (It != FilterOutMap.end() && It->second != DVR)
5061 continue;
5062 }
5063
5064 if (!SunkVariables.insert(DbgUserVariable).second)
5065 continue;
5066
5067 if (DVR->isDbgAssign())
5068 continue;
5069
5070 DVRClones.emplace_back(DVR->clone());
5071 LLVM_DEBUG(dbgs() << "CLONE: " << *DVRClones.back() << '\n');
5072 }
5073
5074 // Perform salvaging without the clones, then sink the clones.
5075 if (DVRClones.empty())
5076 return;
5077
5078 salvageDebugInfoForDbgValues(*I, {}, DbgVariableRecordsToSalvage);
5079
5080 // The clones are in reverse order of original appearance. Assert that the
5081 // head bit is set on the iterator as we _should_ have received it via
5082 // getFirstInsertionPt. Inserting like this will reverse the clone order as
5083 // we'll repeatedly insert at the head, such as:
5084 // DVR-3 (third insertion goes here)
5085 // DVR-2 (second insertion goes here)
5086 // DVR-1 (first insertion goes here)
5087 // Any-Prior-DVRs
5088 // InsertPtInst
5089 assert(InsertPos.getHeadBit());
5090 for (DbgVariableRecord *DVRClone : DVRClones) {
5091 InsertPos->getParent()->insertDbgRecordBefore(DVRClone, InsertPos);
5092 LLVM_DEBUG(dbgs() << "SINK: " << *DVRClone << '\n');
5093 }
5094}
5095
5097 while (!Worklist.isEmpty()) {
5098 // Walk deferred instructions in reverse order, and push them to the
5099 // worklist, which means they'll end up popped from the worklist in-order.
5100 while (Instruction *I = Worklist.popDeferred()) {
5101 // Check to see if we can DCE the instruction. We do this already here to
5102 // reduce the number of uses and thus allow other folds to trigger.
5103 // Note that eraseInstFromFunction() may push additional instructions on
5104 // the deferred worklist, so this will DCE whole instruction chains.
5107 ++NumDeadInst;
5108 continue;
5109 }
5110
5111 Worklist.push(I);
5112 }
5113
5115 if (I == nullptr) continue; // skip null values.
5116
5117 // Check to see if we can DCE the instruction.
5120 ++NumDeadInst;
5121 continue;
5122 }
5123
5124 if (!DebugCounter::shouldExecute(VisitCounter))
5125 continue;
5126
5127 // See if we can trivially sink this instruction to its user if we can
5128 // prove that the successor is not executed more frequently than our block.
5129 // Return the UserBlock if successful.
5130 auto getOptionalSinkBlockForInst =
5131 [this](Instruction *I) -> std::optional<BasicBlock *> {
5132 if (!EnableCodeSinking)
5133 return std::nullopt;
5134
5135 BasicBlock *BB = I->getParent();
5136 BasicBlock *UserParent = nullptr;
5137 unsigned NumUsers = 0;
5138
5139 for (Use &U : I->uses()) {
5140 User *User = U.getUser();
5141 if (User->isDroppable())
5142 continue;
5143 if (NumUsers > MaxSinkNumUsers)
5144 return std::nullopt;
5145
5146 Instruction *UserInst = cast<Instruction>(User);
5147 // Special handling for Phi nodes - get the block the use occurs in.
5148 BasicBlock *UserBB = UserInst->getParent();
5149 if (PHINode *PN = dyn_cast<PHINode>(UserInst))
5150 UserBB = PN->getIncomingBlock(U);
5151 // Bail out if we have uses in different blocks. We don't do any
5152 // sophisticated analysis (i.e finding NearestCommonDominator of these
5153 // use blocks).
5154 if (UserParent && UserParent != UserBB)
5155 return std::nullopt;
5156 UserParent = UserBB;
5157
5158 // Make sure these checks are done only once, naturally we do the checks
5159 // the first time we get the userparent, this will save compile time.
5160 if (NumUsers == 0) {
5161 // Try sinking to another block. If that block is unreachable, then do
5162 // not bother. SimplifyCFG should handle it.
5163 if (UserParent == BB || !DT.isReachableFromEntry(UserParent))
5164 return std::nullopt;
5165
5166 auto *Term = UserParent->getTerminator();
5167 // See if the user is one of our successors that has only one
5168 // predecessor, so that we don't have to split the critical edge.
5169 // Another option where we can sink is a block that ends with a
5170 // terminator that does not pass control to other block (such as
5171 // return or unreachable or resume). In this case:
5172 // - I dominates the User (by SSA form);
5173 // - the User will be executed at most once.
5174 // So sinking I down to User is always profitable or neutral.
5175 if (UserParent->getUniquePredecessor() != BB && !succ_empty(Term))
5176 return std::nullopt;
5177
5178 assert(DT.dominates(BB, UserParent) && "Dominance relation broken?");
5179 }
5180
5181 NumUsers++;
5182 }
5183
5184 // No user or only has droppable users.
5185 if (!UserParent)
5186 return std::nullopt;
5187
5188 return UserParent;
5189 };
5190
5191 auto OptBB = getOptionalSinkBlockForInst(I);
5192 if (OptBB) {
5193 auto *UserParent = *OptBB;
5194 // Okay, the CFG is simple enough, try to sink this instruction.
5195 if (tryToSinkInstruction(I, UserParent)) {
5196 LLVM_DEBUG(dbgs() << "IC: Sink: " << *I << '\n');
5197 MadeIRChange = true;
5198 // We'll add uses of the sunk instruction below, but since
5199 // sinking can expose opportunities for it's *operands* add
5200 // them to the worklist
5201 for (Use &U : I->operands())
5202 if (Instruction *OpI = dyn_cast<Instruction>(U.get()))
5203 Worklist.push(OpI);
5204 }
5205 }
5206
5207 // Now that we have an instruction, try combining it to simplify it.
5210 I, {LLVMContext::MD_dbg, LLVMContext::MD_annotation});
5211
5212#ifndef NDEBUG
5213 std::string OrigI;
5214#endif
5215 LLVM_DEBUG(raw_string_ostream SS(OrigI); I->print(SS););
5216 LLVM_DEBUG(dbgs() << "IC: Visiting: " << OrigI << '\n');
5217
5218 if (Instruction *Result = visit(*I)) {
5219 ++NumCombined;
5220 // Should we replace the old instruction with a new one?
5221 if (Result != I) {
5222 LLVM_DEBUG(dbgs() << "IC: Old = " << *I << '\n'
5223 << " New = " << *Result << '\n');
5224
5225 // We copy the old instruction's DebugLoc to the new instruction, unless
5226 // InstCombine already assigned a DebugLoc to it, in which case we
5227 // should trust the more specifically selected DebugLoc.
5228 if (!Result->getDebugLoc())
5229 Result->setDebugLoc(I->getDebugLoc());
5230 // We also copy annotation metadata to the new instruction.
5231 Result->copyMetadata(*I, LLVMContext::MD_annotation);
5232 // Everything uses the new instruction now.
5233 I->replaceAllUsesWith(Result);
5234
5235 // Move the name to the new instruction first.
5236 Result->takeName(I);
5237
5238 // Insert the new instruction into the basic block...
5239 BasicBlock *InstParent = I->getParent();
5240 BasicBlock::iterator InsertPos = I->getIterator();
5241
5242 // Are we replace a PHI with something that isn't a PHI, or vice versa?
5243 if (isa<PHINode>(Result) != isa<PHINode>(I)) {
5244 // We need to fix up the insertion point.
5245 if (isa<PHINode>(I)) // PHI -> Non-PHI
5246 InsertPos = InstParent->getFirstInsertionPt();
5247 else // Non-PHI -> PHI
5248 InsertPos = InstParent->getFirstNonPHIIt();
5249 }
5250
5251 Result->insertInto(InstParent, InsertPos);
5252
5253 // Push the new instruction and any users onto the worklist.
5255 Worklist.push(Result);
5256
5258 } else {
5259 LLVM_DEBUG(dbgs() << "IC: Mod = " << OrigI << '\n'
5260 << " New = " << *I << '\n');
5261
5262 // If the instruction was modified, it's possible that it is now dead.
5263 // if so, remove it.
5266 } else {
5268 Worklist.push(I);
5269 }
5270 }
5271 MadeIRChange = true;
5272 }
5273 }
5274
5275 Worklist.zap();
5276 return MadeIRChange;
5277}
5278
5279// Track the scopes used by !alias.scope and !noalias. In a function, a
5280// @llvm.experimental.noalias.scope.decl is only useful if that scope is used
5281// by both sets. If not, the declaration of the scope can be safely omitted.
5282// The MDNode of the scope can be omitted as well for the instructions that are
5283// part of this function. We do not do that at this point, as this might become
5284// too time consuming to do.
5286 SmallPtrSet<const MDNode *, 8> UsedAliasScopesAndLists;
5287 SmallPtrSet<const MDNode *, 8> UsedNoAliasScopesAndLists;
5288
5289public:
5291 // This seems to be faster than checking 'mayReadOrWriteMemory()'.
5292 if (!I->hasMetadataOtherThanDebugLoc())
5293 return;
5294
5295 auto Track = [](Metadata *ScopeList, auto &Container) {
5296 const auto *MDScopeList = dyn_cast_or_null<MDNode>(ScopeList);
5297 if (!MDScopeList || !Container.insert(MDScopeList).second)
5298 return;
5299 for (const auto &MDOperand : MDScopeList->operands())
5300 if (auto *MDScope = dyn_cast<MDNode>(MDOperand))
5301 Container.insert(MDScope);
5302 };
5303
5304 Track(I->getMetadata(LLVMContext::MD_alias_scope), UsedAliasScopesAndLists);
5305 Track(I->getMetadata(LLVMContext::MD_noalias), UsedNoAliasScopesAndLists);
5306 }
5307
5309 NoAliasScopeDeclInst *Decl = dyn_cast<NoAliasScopeDeclInst>(Inst);
5310 if (!Decl)
5311 return false;
5312
5313 assert(Decl->use_empty() &&
5314 "llvm.experimental.noalias.scope.decl in use ?");
5315 const MDNode *MDSL = Decl->getScopeList();
5316 assert(MDSL->getNumOperands() == 1 &&
5317 "llvm.experimental.noalias.scope should refer to a single scope");
5318 auto &MDOperand = MDSL->getOperand(0);
5319 if (auto *MD = dyn_cast<MDNode>(MDOperand))
5320 return !UsedAliasScopesAndLists.contains(MD) ||
5321 !UsedNoAliasScopesAndLists.contains(MD);
5322
5323 // Not an MDNode ? throw away.
5324 return true;
5325 }
5326};
5327
5328/// Populate the IC worklist from a function, by walking it in reverse
5329/// post-order and adding all reachable code to the worklist.
5330///
5331/// This has a couple of tricks to make the code faster and more powerful. In
5332/// particular, we constant fold and DCE instructions as we go, to avoid adding
5333/// them to the worklist (this significantly speeds up instcombine on code where
5334/// many instructions are dead or constant). Additionally, if we find a branch
5335/// whose condition is a known constant, we only visit the reachable successors.
5337 bool MadeIRChange = false;
5339 SmallVector<Instruction *, 128> InstrsForInstructionWorklist;
5340 DenseMap<Constant *, Constant *> FoldedConstants;
5341 AliasScopeTracker SeenAliasScopes;
5342
5343 auto HandleOnlyLiveSuccessor = [&](BasicBlock *BB, BasicBlock *LiveSucc) {
5344 for (BasicBlock *Succ : successors(BB))
5345 if (Succ != LiveSucc && DeadEdges.insert({BB, Succ}).second)
5346 for (PHINode &PN : Succ->phis())
5347 for (Use &U : PN.incoming_values())
5348 if (PN.getIncomingBlock(U) == BB && !isa<PoisonValue>(U)) {
5349 U.set(PoisonValue::get(PN.getType()));
5350 MadeIRChange = true;
5351 }
5352 };
5353
5354 for (BasicBlock *BB : RPOT) {
5355 if (!BB->isEntryBlock() && all_of(predecessors(BB), [&](BasicBlock *Pred) {
5356 return DeadEdges.contains({Pred, BB}) || DT.dominates(BB, Pred);
5357 })) {
5358 HandleOnlyLiveSuccessor(BB, nullptr);
5359 continue;
5360 }
5361 LiveBlocks.insert(BB);
5362
5363 for (Instruction &Inst : llvm::make_early_inc_range(*BB)) {
5364 // ConstantProp instruction if trivially constant.
5365 if (!Inst.use_empty() &&
5366 (Inst.getNumOperands() == 0 || isa<Constant>(Inst.getOperand(0))))
5367 if (Constant *C = ConstantFoldInstruction(&Inst, DL, &TLI)) {
5368 LLVM_DEBUG(dbgs() << "IC: ConstFold to: " << *C << " from: " << Inst
5369 << '\n');
5370 Inst.replaceAllUsesWith(C);
5371 ++NumConstProp;
5372 if (isInstructionTriviallyDead(&Inst, &TLI))
5373 Inst.eraseFromParent();
5374 MadeIRChange = true;
5375 continue;
5376 }
5377
5378 // See if we can constant fold its operands.
5379 for (Use &U : Inst.operands()) {
5380 if (!isa<ConstantVector>(U) && !isa<ConstantExpr>(U))
5381 continue;
5382
5383 auto *C = cast<Constant>(U);
5384 Constant *&FoldRes = FoldedConstants[C];
5385 if (!FoldRes)
5386 FoldRes = ConstantFoldConstant(C, DL, &TLI);
5387
5388 if (FoldRes != C) {
5389 LLVM_DEBUG(dbgs() << "IC: ConstFold operand of: " << Inst
5390 << "\n Old = " << *C
5391 << "\n New = " << *FoldRes << '\n');
5392 U = FoldRes;
5393 MadeIRChange = true;
5394 }
5395 }
5396
5397 // Skip processing debug and pseudo intrinsics in InstCombine. Processing
5398 // these call instructions consumes non-trivial amount of time and
5399 // provides no value for the optimization.
5400 if (!Inst.isDebugOrPseudoInst()) {
5401 InstrsForInstructionWorklist.push_back(&Inst);
5402 SeenAliasScopes.analyse(&Inst);
5403 }
5404 }
5405
5406 // If this is a branch or switch on a constant, mark only the single
5407 // live successor. Otherwise assume all successors are live.
5408 Instruction *TI = BB->getTerminator();
5409 if (BranchInst *BI = dyn_cast<BranchInst>(TI); BI && BI->isConditional()) {
5410 if (isa<UndefValue>(BI->getCondition())) {
5411 // Branch on undef is UB.
5412 HandleOnlyLiveSuccessor(BB, nullptr);
5413 continue;
5414 }
5415 if (auto *Cond = dyn_cast<ConstantInt>(BI->getCondition())) {
5416 bool CondVal = Cond->getZExtValue();
5417 HandleOnlyLiveSuccessor(BB, BI->getSuccessor(!CondVal));
5418 continue;
5419 }
5420 } else if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
5421 if (isa<UndefValue>(SI->getCondition())) {
5422 // Switch on undef is UB.
5423 HandleOnlyLiveSuccessor(BB, nullptr);
5424 continue;
5425 }
5426 if (auto *Cond = dyn_cast<ConstantInt>(SI->getCondition())) {
5427 HandleOnlyLiveSuccessor(BB,
5428 SI->findCaseValue(Cond)->getCaseSuccessor());
5429 continue;
5430 }
5431 }
5432 }
5433
5434 // Remove instructions inside unreachable blocks. This prevents the
5435 // instcombine code from having to deal with some bad special cases, and
5436 // reduces use counts of instructions.
5437 for (BasicBlock &BB : F) {
5438 if (LiveBlocks.count(&BB))
5439 continue;
5440
5441 unsigned NumDeadInstInBB;
5442 unsigned NumDeadDbgInstInBB;
5443 std::tie(NumDeadInstInBB, NumDeadDbgInstInBB) =
5445
5446 MadeIRChange |= NumDeadInstInBB + NumDeadDbgInstInBB > 0;
5447 NumDeadInst += NumDeadInstInBB;
5448 }
5449
5450 // Once we've found all of the instructions to add to instcombine's worklist,
5451 // add them in reverse order. This way instcombine will visit from the top
5452 // of the function down. This jives well with the way that it adds all uses
5453 // of instructions to the worklist after doing a transformation, thus avoiding
5454 // some N^2 behavior in pathological cases.
5455 Worklist.reserve(InstrsForInstructionWorklist.size());
5456 for (Instruction *Inst : reverse(InstrsForInstructionWorklist)) {
5457 // DCE instruction if trivially dead. As we iterate in reverse program
5458 // order here, we will clean up whole chains of dead instructions.
5459 if (isInstructionTriviallyDead(Inst, &TLI) ||
5460 SeenAliasScopes.isNoAliasScopeDeclDead(Inst)) {
5461 ++NumDeadInst;
5462 LLVM_DEBUG(dbgs() << "IC: DCE: " << *Inst << '\n');
5463 salvageDebugInfo(*Inst);
5464 Inst->eraseFromParent();
5465 MadeIRChange = true;
5466 continue;
5467 }
5468
5469 Worklist.push(Inst);
5470 }
5471
5472 return MadeIRChange;
5473}
5474
5476 // Collect backedges.
5478 for (BasicBlock *BB : RPOT) {
5479 Visited.insert(BB);
5480 for (BasicBlock *Succ : successors(BB))
5481 if (Visited.contains(Succ))
5482 BackEdges.insert({BB, Succ});
5483 }
5484 ComputedBackEdges = true;
5485}
5486
5492 const InstCombineOptions &Opts) {
5493 auto &DL = F.getDataLayout();
5494 bool VerifyFixpoint = Opts.VerifyFixpoint &&
5495 !F.hasFnAttribute("instcombine-no-verify-fixpoint");
5496
5497 /// Builder - This is an IRBuilder that automatically inserts new
5498 /// instructions into the worklist when they are created.
5500 F.getContext(), TargetFolder(DL),
5501 IRBuilderCallbackInserter([&Worklist, &AC](Instruction *I) {
5502 Worklist.add(I);
5503 if (auto *Assume = dyn_cast<AssumeInst>(I))
5504 AC.registerAssumption(Assume);
5505 }));
5506
5508
5509 // Lower dbg.declare intrinsics otherwise their value may be clobbered
5510 // by instcombiner.
5511 bool MadeIRChange = false;
5513 MadeIRChange = LowerDbgDeclare(F);
5514
5515 // Iterate while there is work to do.
5516 unsigned Iteration = 0;
5517 while (true) {
5518 ++Iteration;
5519
5520 if (Iteration > Opts.MaxIterations && !VerifyFixpoint) {
5521 LLVM_DEBUG(dbgs() << "\n\n[IC] Iteration limit #" << Opts.MaxIterations
5522 << " on " << F.getName()
5523 << " reached; stopping without verifying fixpoint\n");
5524 break;
5525 }
5526
5527 ++NumWorklistIterations;
5528 LLVM_DEBUG(dbgs() << "\n\nINSTCOMBINE ITERATION #" << Iteration << " on "
5529 << F.getName() << "\n");
5530
5531 InstCombinerImpl IC(Worklist, Builder, F.hasMinSize(), AA, AC, TLI, TTI, DT,
5532 ORE, BFI, BPI, PSI, DL, RPOT);
5534 bool MadeChangeInThisIteration = IC.prepareWorklist(F);
5535 MadeChangeInThisIteration |= IC.run();
5536 if (!MadeChangeInThisIteration)
5537 break;
5538
5539 MadeIRChange = true;
5540 if (Iteration > Opts.MaxIterations) {
5542 "Instruction Combining on " + Twine(F.getName()) +
5543 " did not reach a fixpoint after " + Twine(Opts.MaxIterations) +
5544 " iterations. " +
5545 "Use 'instcombine<no-verify-fixpoint>' or function attribute "
5546 "'instcombine-no-verify-fixpoint' to suppress this error.",
5547 /*GenCrashDiag=*/false);
5548 }
5549 }
5550
5551 if (Iteration == 1)
5552 ++NumOneIteration;
5553 else if (Iteration == 2)
5554 ++NumTwoIterations;
5555 else if (Iteration == 3)
5556 ++NumThreeIterations;
5557 else
5558 ++NumFourOrMoreIterations;
5559
5560 return MadeIRChange;
5561}
5562
5564
5566 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
5567 static_cast<PassInfoMixin<InstCombinePass> *>(this)->printPipeline(
5568 OS, MapClassName2PassName);
5569 OS << '<';
5570 OS << "max-iterations=" << Options.MaxIterations << ";";
5571 OS << (Options.VerifyFixpoint ? "" : "no-") << "verify-fixpoint";
5572 OS << '>';
5573}
5574
5575char InstCombinePass::ID = 0;
5576
5579 auto &LRT = AM.getResult<LastRunTrackingAnalysis>(F);
5580 // No changes since last InstCombine pass, exit early.
5581 if (LRT.shouldSkip(&ID))
5582 return PreservedAnalyses::all();
5583
5584 auto &AC = AM.getResult<AssumptionAnalysis>(F);
5585 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
5586 auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
5588 auto &TTI = AM.getResult<TargetIRAnalysis>(F);
5589
5590 auto *AA = &AM.getResult<AAManager>(F);
5591 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
5592 ProfileSummaryInfo *PSI =
5593 MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
5594 auto *BFI = (PSI && PSI->hasProfileSummary()) ?
5595 &AM.getResult<BlockFrequencyAnalysis>(F) : nullptr;
5597
5598 if (!combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE,
5599 BFI, BPI, PSI, Options)) {
5600 // No changes, all analyses are preserved.
5601 LRT.update(&ID, /*Changed=*/false);
5602 return PreservedAnalyses::all();
5603 }
5604
5605 // Mark all the analyses that instcombine updates as preserved.
5607 LRT.update(&ID, /*Changed=*/true);
5610 return PA;
5611}
5612
5614 AU.setPreservesCFG();
5627}
5628
5630 if (skipFunction(F))
5631 return false;
5632
5633 // Required analyses.
5634 auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
5635 auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
5636 auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
5637 auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
5638 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
5639 auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
5640
5641 // Optional analyses.
5642 ProfileSummaryInfo *PSI =
5643 &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
5644 BlockFrequencyInfo *BFI =
5645 (PSI && PSI->hasProfileSummary()) ?
5646 &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI() :
5647 nullptr;
5648 BranchProbabilityInfo *BPI = nullptr;
5649 if (auto *WrapperPass =
5650 getAnalysisIfAvailable<BranchProbabilityInfoWrapperPass>())
5651 BPI = &WrapperPass->getBPI();
5652
5653 return combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE,
5654 BFI, BPI, PSI, InstCombineOptions());
5655}
5656
5658
5661}
5662
5664 "Combine redundant instructions", false, false)
5676
5677// Initialization Routines
5680}
5681
5683 return new InstructionCombiningPass();
5684}
AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
static const Function * getParent(const Value *V)
This is the interface for LLVM's primary stateless and local alias analysis.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
Definition: DebugCounter.h:190
#define LLVM_DEBUG(...)
Definition: Debug.h:106
This file defines the DenseMap class.
uint64_t Size
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
This is the interface for a simple mod/ref and alias analysis over globals.
Hexagon Common GEP
Hexagon Vector Combine
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This header defines various interfaces for pass management in LLVM.
This defines the Use class.
iv Induction Variable Users
Definition: IVUsers.cpp:48
static bool leftDistributesOverRight(Instruction::BinaryOps LOp, bool HasNUW, bool HasNSW, Intrinsic::ID ROp)
Return whether "X LOp (Y ROp Z)" is always equal to "(X LOp Y) ROp (X LOp Z)".
This file provides internal interfaces used to implement the InstCombine.
This file provides the primary interface to the instcombine pass.
static Value * simplifySwitchOnSelectUsingRanges(SwitchInst &SI, SelectInst *Select, bool IsTrueArm)
static bool isUsedWithinShuffleVector(Value *V)
static bool isNeverEqualToUnescapedAlloc(Value *V, const TargetLibraryInfo &TLI, Instruction *AI)
static bool shorter_filter(const Value *LHS, const Value *RHS)
static Instruction * foldSelectGEP(GetElementPtrInst &GEP, InstCombiner::BuilderTy &Builder)
Thread a GEP operation with constant indices through the constant true/false arms of a select.
static bool shouldMergeGEPs(GEPOperator &GEP, GEPOperator &Src)
static cl::opt< unsigned > MaxArraySize("instcombine-maxarray-size", cl::init(1024), cl::desc("Maximum array size considered when doing a combine"))
static cl::opt< unsigned > ShouldLowerDbgDeclare("instcombine-lower-dbg-declare", cl::Hidden, cl::init(true))
static bool hasNoSignedWrap(BinaryOperator &I)
static bool simplifyAssocCastAssoc(BinaryOperator *BinOp1, InstCombinerImpl &IC)
Combine constant operands of associative operations either before or after a cast to eliminate one of...
static bool combineInstructionsOverFunction(Function &F, InstructionWorklist &Worklist, AliasAnalysis *AA, AssumptionCache &AC, TargetLibraryInfo &TLI, TargetTransformInfo &TTI, DominatorTree &DT, OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI, BranchProbabilityInfo *BPI, ProfileSummaryInfo *PSI, const InstCombineOptions &Opts)
static Value * simplifyInstructionWithPHI(Instruction &I, PHINode *PN, Value *InValue, BasicBlock *InBB, const DataLayout &DL, const SimplifyQuery SQ)
static bool shouldCanonicalizeGEPToPtrAdd(GetElementPtrInst &GEP)
Return true if we should canonicalize the gep to an i8 ptradd.
static void ClearSubclassDataAfterReassociation(BinaryOperator &I)
Conservatively clears subclassOptionalData after a reassociation or commutation.
static bool isAllocSiteRemovable(Instruction *AI, SmallVectorImpl< WeakTrackingVH > &Users, const TargetLibraryInfo &TLI)
static Value * getIdentityValue(Instruction::BinaryOps Opcode, Value *V)
This function returns identity value for given opcode, which can be used to factor patterns like (X *...
static std::optional< std::pair< Value *, Value * > > matchSymmetricPhiNodesPair(PHINode *LHS, PHINode *RHS)
static Value * foldOperationIntoSelectOperand(Instruction &I, SelectInst *SI, Value *NewOp, InstCombiner &IC)
static Instruction * canonicalizeGEPOfConstGEPI8(GetElementPtrInst &GEP, GEPOperator *Src, InstCombinerImpl &IC)
static Instruction * tryToMoveFreeBeforeNullTest(CallInst &FI, const DataLayout &DL)
Move the call to free before a NULL test.
static Value * simplifyOperationIntoSelectOperand(Instruction &I, SelectInst *SI, bool IsTrueArm)
static bool rightDistributesOverLeft(Instruction::BinaryOps LOp, Instruction::BinaryOps ROp)
Return whether "(X LOp Y) ROp Z" is always equal to "(X ROp Z) LOp (Y ROp Z)".
static Value * tryFactorization(BinaryOperator &I, const SimplifyQuery &SQ, InstCombiner::BuilderTy &Builder, Instruction::BinaryOps InnerOpcode, Value *A, Value *B, Value *C, Value *D)
This tries to simplify binary operations by factorizing out common terms (e.
static bool isRemovableWrite(CallBase &CB, Value *UsedV, const TargetLibraryInfo &TLI)
Given a call CB which uses an address UsedV, return true if we can prove the call's only possible eff...
static Instruction::BinaryOps getBinOpsForFactorization(Instruction::BinaryOps TopOpcode, BinaryOperator *Op, Value *&LHS, Value *&RHS, BinaryOperator *OtherOp)
This function predicates factorization using distributive laws.
static bool hasNoUnsignedWrap(BinaryOperator &I)
static bool SoleWriteToDeadLocal(Instruction *I, TargetLibraryInfo &TLI)
Check for case where the call writes to an otherwise dead alloca.
static cl::opt< unsigned > MaxSinkNumUsers("instcombine-max-sink-users", cl::init(32), cl::desc("Maximum number of undroppable users for instruction sinking"))
static Instruction * foldGEPOfPhi(GetElementPtrInst &GEP, PHINode *PN, IRBuilderBase &Builder)
static bool isCatchAll(EHPersonality Personality, Constant *TypeInfo)
Return 'true' if the given typeinfo will match anything.
static cl::opt< bool > EnableCodeSinking("instcombine-code-sinking", cl::desc("Enable code sinking"), cl::init(true))
static bool maintainNoSignedWrap(BinaryOperator &I, Value *B, Value *C)
static GEPNoWrapFlags getMergedGEPNoWrapFlags(GEPOperator &GEP1, GEPOperator &GEP2)
Determine nowrap flags for (gep (gep p, x), y) to (gep p, (x + y)) transform.
static LVOptions Options
Definition: LVOptions.cpp:25
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file contains the declarations for metadata subclasses.
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static bool IsSelect(MachineInstr &MI)
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:57
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
unsigned OpIndex
raw_pwrite_stream & OS
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:166
static unsigned getScalarSizeInBits(Type *Ty)
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:39
This pass exposes codegen information to IR-level passes.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:191
Value * RHS
Value * LHS
static const uint32_t IV[8]
Definition: blake3_impl.h:78
bool isNoAliasScopeDeclDead(Instruction *Inst)
void analyse(Instruction *I)
A manager for alias analyses.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
Class for arbitrary precision integers.
Definition: APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:234
static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
Definition: APInt.cpp:1732
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition: APInt.h:423
static void sdivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Definition: APInt.cpp:1864
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:910
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition: APInt.h:371
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition: APInt.h:380
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1468
APInt sadd_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1902
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition: APInt.h:827
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:334
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition: APInt.h:1150
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:440
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:306
APInt ssub_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1915
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:851
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:253
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
Definition: PassManager.h:429
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:410
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:256
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:168
Class to represent array types.
Definition: DerivedTypes.h:395
uint64_t getNumElements() const
Definition: DerivedTypes.h:407
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Type * getElementType() const
Definition: DerivedTypes.h:408
A function analysis which provides an AssumptionCache.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
void registerAssumption(AssumeInst *CI)
Add an @llvm.assume intrinsic to this function's cache.
uint64_t getDereferenceableBytes() const
Returns the number of dereferenceable bytes from the dereferenceable attribute.
Definition: Attributes.cpp:439
bool isValid() const
Return true if the attribute is any kind of attribute.
Definition: Attributes.h:208
Legacy wrapper pass to provide the BasicAAResult object.
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:517
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:416
iterator_range< filter_iterator< BasicBlock::const_iterator, std::function< bool(const Instruction &)> > > instructionsWithoutDebug(bool SkipPseudoOp=true) const
Return a const iterator range over the instructions in the block, skipping any debug instructions.
Definition: BasicBlock.cpp:250
InstListType::const_iterator getFirstNonPHIIt() const
Iterator returning form of getFirstNonPHI.
Definition: BasicBlock.cpp:374
const Instruction & front() const
Definition: BasicBlock.h:471
bool isEntryBlock() const
Return true if this is the entry block of the containing function.
Definition: BasicBlock.cpp:571
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:459
const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
Definition: BasicBlock.cpp:467
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:177
const_iterator getFirstNonPHIOrDbgOrAlloca() const
Returns an iterator to the first instruction in this block that is not a PHINode, a debug intrinsic,...
Definition: BasicBlock.cpp:430
size_t size() const
Definition: BasicBlock.h:469
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:239
static BinaryOperator * CreateNeg(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Helper functions to construct and inspect unary operations (NEG and NOT) via binary operators SUB and...
BinaryOps getOpcode() const
Definition: InstrTypes.h:370
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
static BinaryOperator * CreateNUW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name="")
Definition: InstrTypes.h:293
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Conditional or Unconditional Branch instruction.
void swapSuccessors()
Swap the successors of this branch instruction.
bool isConditional() const
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
Analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
void swapSuccEdgesProbabilities(const BasicBlock *Src)
Swap outgoing edges probabilities for Src with branch terminator.
Represents analyses that only rely on functions' control flow.
Definition: Analysis.h:72
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1120
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition: InstrTypes.h:1428
bool doesNotThrow() const
Determine if the call cannot unwind.
Definition: InstrTypes.h:1923
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1294
AttributeList getAttributes() const
Return the attributes for this call.
Definition: InstrTypes.h:1425
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:673
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:696
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:698
@ ICMP_EQ
equal
Definition: InstrTypes.h:694
@ ICMP_NE
not equal
Definition: InstrTypes.h:695
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition: InstrTypes.h:825
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition: InstrTypes.h:787
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
Definition: CmpPredicate.h:22
ConstantArray - Constant Array Declarations.
Definition: Constants.h:427
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1312
A vector constant whose element type is a simple 1/2/4/8-byte integer or float/double,...
Definition: Constants.h:770
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2644
static Constant * getNot(Constant *C)
Definition: Constants.cpp:2631
static Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2637
static Constant * getBinOpIdentity(unsigned Opcode, Type *Ty, bool AllowRHSConstant=false, bool NSZ=false)
Return the identity constant for a binary opcode.
Definition: Constants.cpp:2691
static Constant * getNeg(Constant *C, bool HasNSW=false)
Definition: Constants.cpp:2625
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:866
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:873
static ConstantInt * getBool(LLVMContext &Context, bool V)
Definition: Constants.cpp:880
This class represents a range of values.
Definition: ConstantRange.h:47
bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const
Set up Pred and RHS such that ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.
static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
bool contains(const APInt &Val) const
Return true if the specified value is in the set.
static ConstantRange makeExactNoWrapRegion(Instruction::BinaryOps BinOp, const APInt &Other, unsigned NoWrapKind)
Produce the range that contains X if and only if "X BinOp Other" does not wrap.
Constant Vector Declarations.
Definition: Constants.h:511
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:1421
This is an important base class in LLVM.
Definition: Constant.h:42
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
Definition: Constants.cpp:403
static Constant * replaceUndefsWith(Constant *C, Constant *Replacement)
Try to replace undefined constant C or undefined elements in C with Replacement.
Definition: Constants.cpp:784
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:420
const Constant * stripPointerCasts() const
Definition: Constant.h:218
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:373
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
Definition: Constants.cpp:435
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition: Constants.cpp:90
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
SmallVector< APInt > getGEPIndicesForOffset(Type *&ElemTy, APInt &Offset) const
Get GEP indices to access Offset inside ElemTy.
Definition: DataLayout.cpp:971
bool isLegalInteger(uint64_t Width) const
Returns true if the specified type is known to be a native integer type supported by the CPU.
Definition: DataLayout.h:219
unsigned getIndexTypeSizeInBits(Type *Ty) const
Layout size of the index used in GEP calculation.
Definition: DataLayout.cpp:754
IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
Definition: DataLayout.cpp:878
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:457
unsigned getIndexSizeInBits(unsigned AS) const
Size in bits of index used for address calculation in getelementptr.
Definition: DataLayout.h:369
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:617
int64_t getIndexedOffsetInType(Type *ElemTy, ArrayRef< Value * > Indices) const
Returns the offset from the beginning of the type for the specified indices.
Definition: DataLayout.cpp:893
This is the common base class for debug info intrinsics for variables.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
static bool shouldExecute(unsigned CounterName)
Definition: DebugCounter.h:87
Identifies a unique instance of a variable.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: DenseMap.h:194
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:156
bool empty() const
Definition: DenseMap.h:98
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:211
void registerBranch(BranchInst *BI)
Add a branch condition to the cache.
Analysis pass which computes a DominatorTree.
Definition: Dominators.h:279
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:317
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
Definition: Dominators.cpp:321
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Definition: Dominators.cpp:122
This instruction extracts a struct member or array element value from an aggregate value.
ArrayRef< unsigned > getIndices() const
iterator_range< idx_iterator > indices() const
idx_iterator idx_end() const
static ExtractValueInst * Create(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
idx_iterator idx_begin() const
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition: Operator.h:205
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:20
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:310
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:178
const BasicBlock & getEntryBlock() const
Definition: Function.h:809
Represents flags for the getelementptr instruction/expression.
GEPNoWrapFlags withoutNoUnsignedSignedWrap() const
static GEPNoWrapFlags noUnsignedWrap()
GEPNoWrapFlags intersectForOffsetAdd(GEPNoWrapFlags Other) const
Given (gep (gep p, x), y), determine the nowrap flags for (gep p, x+y).
GEPNoWrapFlags withoutNoUnsignedWrap() const
GEPNoWrapFlags getNoWrapFlags() const
Definition: Operator.h:430
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:933
static Type * getTypeAtIndex(Type *Ty, Value *Idx)
Return the type of the element at the given index of an indexable type.
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Definition: Instructions.h:956
static GetElementPtrInst * CreateInBounds(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Create an "inbounds" getelementptr.
Definition: Instructions.h:980
Legacy wrapper pass to provide the GlobalsAAResult object.
This instruction compares its operands according to the predicate given to the constructor.
CmpPredicate getCmpPredicate() const
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:108
Value * CreateLogicalOp(Instruction::BinaryOps Opc, Value *Cond1, Value *Cond2, const Twine &Name="")
Definition: IRBuilder.h:1694
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition: IRBuilder.h:2549
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.cpp:1043
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2039
Value * CreateFreeze(Value *V, const Twine &Name="")
Definition: IRBuilder.h:2568
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Definition: IRBuilder.h:1981
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition: IRBuilder.h:325
Value * CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
Definition: IRBuilder.h:1876
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Definition: IRBuilder.h:1868
void CollectMetadataToCopy(Instruction *Src, ArrayRef< unsigned > MetadataKinds)
Collect metadata with IDs MetadataKinds from Src which should be added to all created instructions.
Definition: IRBuilder.h:247
Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Definition: IRBuilder.cpp:879
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Definition: IRBuilder.cpp:890
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:500
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2398
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2429
Value * CreateNot(Value *V, const Twine &Name="")
Definition: IRBuilder.h:1751
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1381
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition: IRBuilder.h:1792
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Definition: IRBuilder.h:2527
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1512
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1364
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Definition: IRBuilder.h:2013
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1665
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:2219
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition: IRBuilder.h:194
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1493
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1556
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2374
Value * CreateLogicalOr(Value *Cond1, Value *Cond2, const Twine &Name="")
Definition: IRBuilder.h:1688
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Definition: IRBuilder.h:530
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
Definition: IRBuilder.h:516
Provides an 'InsertHelper' that calls a user-provided callback after performing the default insertion...
Definition: IRBuilder.h:74
This instruction inserts a struct field of array element value into an aggregate value.
static InsertValueInst * Create(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
InstCombinePass(InstCombineOptions Opts={})
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false)
Given an instruction with a select as one operand and a constant as the other operand,...
Instruction * foldBinOpOfSelectAndCastOfSelectCondition(BinaryOperator &I)
Tries to simplify binops of select and cast of the select condition.
Instruction * foldBinOpIntoSelectOrPhi(BinaryOperator &I)
This is a convenience wrapper function for the above two functions.
bool SimplifyAssociativeOrCommutative(BinaryOperator &I)
Performs a few simplifications for operators which are associative or commutative.
Instruction * visitGEPOfGEP(GetElementPtrInst &GEP, GEPOperator *Src)
Value * foldUsingDistributiveLaws(BinaryOperator &I)
Tries to simplify binary operations which some other binary operation distributes over.
Instruction * foldBinOpShiftWithShift(BinaryOperator &I)
Instruction * visitUnreachableInst(UnreachableInst &I)
Instruction * foldOpIntoPhi(Instruction &I, PHINode *PN, bool AllowMultipleUses=false)
Given a binary operator, cast instruction, or select which has a PHI node as operand #0,...
void handleUnreachableFrom(Instruction *I, SmallVectorImpl< BasicBlock * > &Worklist)
Value * SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &PoisonElts, unsigned Depth=0, bool AllowMultipleUsers=false) override
The specified value produces a vector with any number of elements.
Instruction * visitFreeze(FreezeInst &I)
void handlePotentiallyDeadBlocks(SmallVectorImpl< BasicBlock * > &Worklist)
bool prepareWorklist(Function &F)
Perform early cleanup and prepare the InstCombine worklist.
Instruction * visitFree(CallInst &FI, Value *FreedOp)
Instruction * visitExtractValueInst(ExtractValueInst &EV)
void handlePotentiallyDeadSuccessors(BasicBlock *BB, BasicBlock *LiveSucc)
Instruction * visitUnconditionalBranchInst(BranchInst &BI)
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Instruction * visitLandingPadInst(LandingPadInst &LI)
Instruction * visitReturnInst(ReturnInst &RI)
Instruction * visitSwitchInst(SwitchInst &SI)
Instruction * foldBinopWithPhiOperands(BinaryOperator &BO)
For a binary operator with 2 phi operands, try to hoist the binary operation before the phi.
Constant * getLosslessTrunc(Constant *C, Type *TruncTy, unsigned ExtOp)
Value * SimplifyDemandedUseFPClass(Value *V, FPClassTest DemandedMask, KnownFPClass &Known, unsigned Depth, Instruction *CxtI)
Attempts to replace V with a simpler value based on the demanded floating-point classes.
bool mergeStoreIntoSuccessor(StoreInst &SI)
Try to transform: if () { *P = v1; } else { *P = v2 } or: *P = v1; if () { *P = v2; } into a phi node...
Instruction * tryFoldInstWithCtpopWithNot(Instruction *I)
void tryToSinkInstructionDbgValues(Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock, BasicBlock *DestBlock, SmallVectorImpl< DbgVariableIntrinsic * > &DbgUsers)
void CreateNonTerminatorUnreachable(Instruction *InsertAt)
Create and insert the idiom we use to indicate a block is unreachable without having to rewrite the C...
Value * pushFreezeToPreventPoisonFromPropagating(FreezeInst &FI)
bool run()
Run the combiner over the entire worklist until it is empty.
Instruction * foldVectorBinop(BinaryOperator &Inst)
Canonicalize the position of binops relative to shufflevector.
bool removeInstructionsBeforeUnreachable(Instruction &I)
Value * SimplifySelectsFeedingBinaryOp(BinaryOperator &I, Value *LHS, Value *RHS)
void tryToSinkInstructionDbgVariableRecords(Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock, BasicBlock *DestBlock, SmallVectorImpl< DbgVariableRecord * > &DPUsers)
void addDeadEdge(BasicBlock *From, BasicBlock *To, SmallVectorImpl< BasicBlock * > &Worklist)
Instruction * visitAllocSite(Instruction &FI)
Instruction * visitGetElementPtrInst(GetElementPtrInst &GEP)
Instruction * visitBranchInst(BranchInst &BI)
Value * tryFactorizationFolds(BinaryOperator &I)
This tries to simplify binary operations by factorizing out common terms (e.
Instruction * foldFreezeIntoRecurrence(FreezeInst &I, PHINode *PN)
bool tryToSinkInstruction(Instruction *I, BasicBlock *DestBlock)
Try to move the specified instruction from its current block into the beginning of DestBlock,...
bool freezeOtherUses(FreezeInst &FI)
void freelyInvertAllUsersOf(Value *V, Value *IgnoredUser=nullptr)
Freely adapt every user of V as-if V was changed to !V.
The core instruction combiner logic.
Definition: InstCombiner.h:48
SimplifyQuery SQ
Definition: InstCombiner.h:77
const DataLayout & getDataLayout() const
Definition: InstCombiner.h:343
bool isFreeToInvert(Value *V, bool WillInvertAllUses, bool &DoesConsume)
Return true if the specified value is free to invert (apply ~ to).
Definition: InstCombiner.h:234
static unsigned getComplexity(Value *V)
Assign a complexity or rank value to LLVM Values.
Definition: InstCombiner.h:143
TargetLibraryInfo & TLI
Definition: InstCombiner.h:74
Instruction * InsertNewInstBefore(Instruction *New, BasicBlock::iterator Old)
Inserts an instruction New before instruction Old.
Definition: InstCombiner.h:374
AAResults * AA
Definition: InstCombiner.h:70
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
Definition: InstCombiner.h:394
uint64_t MaxArraySizeForCombine
Maximum size of array considered when transforming.
Definition: InstCombiner.h:56
static bool shouldAvoidAbsorbingNotIntoSelect(const SelectInst &SI)
Definition: InstCombiner.h:193
void replaceUse(Use &U, Value *NewValue)
Replace use and add the previously used value to the worklist.
Definition: InstCombiner.h:426
static bool isCanonicalPredicate(CmpPredicate Pred)
Predicate canonicalization reduces the number of patterns that need to be matched by other transforms...
Definition: InstCombiner.h:160
InstructionWorklist & Worklist
A worklist of the instructions that need to be simplified.
Definition: InstCombiner.h:65
Instruction * InsertNewInstWith(Instruction *New, BasicBlock::iterator Old)
Same as InsertNewInstBefore, but also sets the debug loc.
Definition: InstCombiner.h:383
BranchProbabilityInfo * BPI
Definition: InstCombiner.h:80
ReversePostOrderTraversal< BasicBlock * > & RPOT
Definition: InstCombiner.h:84
const DataLayout & DL
Definition: InstCombiner.h:76
unsigned ComputeNumSignBits(const Value *Op, unsigned Depth=0, const Instruction *CxtI=nullptr) const
Definition: InstCombiner.h:461
DomConditionCache DC
Definition: InstCombiner.h:82
const bool MinimizeSize
Definition: InstCombiner.h:68
std::optional< Instruction * > targetInstCombineIntrinsic(IntrinsicInst &II)
void addToWorklist(Instruction *I)
Definition: InstCombiner.h:338
Value * getFreelyInvertedImpl(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume, unsigned Depth)
Return nonnull value if V is free to invert under the condition of WillInvertAllUses.
SmallDenseSet< std::pair< const BasicBlock *, const BasicBlock * >, 8 > BackEdges
Backedges, used to avoid pushing instructions across backedges in cases where this may result in infi...
Definition: InstCombiner.h:97
std::optional< Value * > targetSimplifyDemandedVectorEltsIntrinsic(IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3, std::function< void(Instruction *, unsigned, APInt, APInt &)> SimplifyAndSetOp)
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
Definition: InstCombiner.h:418
DominatorTree & DT
Definition: InstCombiner.h:75
static Constant * getSafeVectorConstantForBinop(BinaryOperator::BinaryOps Opcode, Constant *In, bool IsRHSConstant)
Some binary operators require special handling to avoid poison and undefined behavior.
Definition: InstCombiner.h:286
SmallDenseSet< std::pair< BasicBlock *, BasicBlock * >, 8 > DeadEdges
Edges that are known to never be taken.
Definition: InstCombiner.h:89
std::optional< Value * > targetSimplifyDemandedUseBitsIntrinsic(IntrinsicInst &II, APInt DemandedMask, KnownBits &Known, bool &KnownBitsComputed)
void computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth, const Instruction *CxtI) const
Definition: InstCombiner.h:439
BuilderTy & Builder
Definition: InstCombiner.h:61
bool isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const
Value * getFreelyInverted(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume)
Definition: InstCombiner.h:215
bool isBackEdge(const BasicBlock *From, const BasicBlock *To)
Definition: InstCombiner.h:364
void visit(Iterator Start, Iterator End)
Definition: InstVisitor.h:87
The legacy pass manager's instcombine pass.
Definition: InstCombine.h:66
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool runOnFunction(Function &F) override
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
InstructionWorklist - This is the worklist management logic for InstCombine and other simplification ...
void pushUsersToWorkList(Instruction &I)
When an instruction is simplified, add all users of the instruction to the work lists because they mi...
void add(Instruction *I)
Add instruction to the worklist.
void push(Instruction *I)
Push the instruction onto the worklist stack.
void zap()
Check that the worklist is empty and nuke the backing store for the map.
static bool isBitwiseLogicOp(unsigned Opcode)
Determine if the Opcode is and/or/xor.
Definition: Instruction.h:328
void copyIRFlags(const Value *V, bool IncludeWrapFlags=true)
Convenience method to copy supported exact, fast-math, and (optionally) wrapping flags from V to this...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:475
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:68
void setAAMetadata(const AAMDNodes &N)
Sets the AA metadata on this instruction from the AAMDNodes structure.
Definition: Metadata.cpp:1764
bool isAssociative() const LLVM_READONLY
Return true if the instruction is associative:
bool isCommutative() const LLVM_READONLY
Return true if the instruction is commutative:
void setFastMathFlags(FastMathFlags FMF)
Convenience function for setting multiple fast-math flags on this instruction, which must be an opera...
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:72
bool isTerminator() const
Definition: Instruction.h:277
void dropUBImplyingAttrsAndMetadata()
Drop any attributes or metadata that can cause immediate undefined behavior.
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
bool willReturn() const LLVM_READONLY
Return true if the instruction will return (unwinding is considered as a form of returning control fl...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:274
bool isBitwiseLogicOp() const
Return true if this is and/or/xor.
Definition: Instruction.h:333
bool isShift() const
Definition: Instruction.h:282
void dropPoisonGeneratingFlags()
Drops flags that may cause this instruction to evaluate to poison despite having non-poison inputs.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:472
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
bool isIntDivRem() const
Definition: Instruction.h:280
Class to represent integer types.
Definition: DerivedTypes.h:42
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:311
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:48
Invoke instruction.
static InvokeInst * Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value * > Args, const Twine &NameStr, InsertPosition InsertBefore=nullptr)
The landingpad instruction holds all of the information necessary to generate correct exception handl...
bool isCleanup() const
Return 'true' if this landingpad instruction is a cleanup.
unsigned getNumClauses() const
Get the number of clauses for this landing pad.
static LandingPadInst * Create(Type *RetTy, unsigned NumReservedClauses, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedClauses is a hint for the number of incoming clauses that this landingpad w...
void addClause(Constant *ClauseVal)
Add a catch or filter clause to the landing pad.
bool isCatch(unsigned Idx) const
Return 'true' if the clause and index Idx is a catch clause.
bool isFilter(unsigned Idx) const
Return 'true' if the clause and index Idx is a filter clause.
Constant * getClause(unsigned Idx) const
Get the value of the clause at index Idx.
void setCleanup(bool V)
Indicate that this landingpad instruction is a cleanup.
A function/module analysis which provides an empty LastRunTrackingInfo.
This is an alternative analysis pass to BlockFrequencyInfoWrapperPass.
static void getLazyBFIAnalysisUsage(AnalysisUsage &AU)
Helper for client passes to set up the analysis usage on behalf of this pass.
An instruction for reading from memory.
Definition: Instructions.h:176
Metadata node.
Definition: Metadata.h:1069
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1430
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1436
Tracking metadata reference owned by Metadata.
Definition: Metadata.h:891
This is the common base class for memset/memcpy/memmove.
static MemoryLocation getForDest(const MemIntrinsic *MI)
Return a location representing the destination of a memory set or transfer.
Root of the metadata hierarchy.
Definition: Metadata.h:62
This class represents min/max intrinsics.
Value * getLHS() const
Value * getRHS() const
static ICmpInst::Predicate getPredicate(Intrinsic::ID ID)
Returns the comparison predicate underlying the intrinsic.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
MDNode * getScopeList() const
OptimizationRemarkEmitter legacy analysis pass.
The optimization diagnostic interface.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Definition: PassManager.h:692
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition: Operator.h:77
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property.
Definition: Operator.h:110
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property.
Definition: Operator.h:104
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
Definition: PassRegistry.h:37
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
Definition: Constants.h:1460
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1878
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:111
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:117
void preserveSet()
Mark an analysis set as preserved.
Definition: Analysis.h:146
void preserve()
Mark an analysis as preserved.
Definition: Analysis.h:131
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
bool hasProfileSummary() const
Returns true if profile summary is available.
A global registry used in conjunction with static constructors to make pluggable components (like tar...
Definition: Registry.h:44
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, InsertPosition InsertBefore=nullptr)
This class represents a cast from signed integer to floating point.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, Instruction *MDFrom=nullptr)
This instruction constructs a fixed permutation of two input vectors.
size_type size() const
Definition: SmallPtrSet.h:94
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:452
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:384
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:458
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:519
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:132
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:181
bool empty() const
Definition: SmallVector.h:81
size_t size() const
Definition: SmallVector.h:78
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:573
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:937
void reserve(size_type N)
Definition: SmallVector.h:663
iterator erase(const_iterator CI)
Definition: SmallVector.h:737
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:683
typename SuperClass::iterator iterator
Definition: SmallVector.h:577
void push_back(const T &Elt)
Definition: SmallVector.h:413
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
An instruction for storing to memory.
Definition: Instructions.h:292
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
Multiway switch.
TargetFolder - Create constants with target dependent folding.
Definition: TargetFolder.h:34
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
bool has(LibFunc F) const
Tests whether a library function is available.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
Wrapper pass for TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
std::optional< Instruction * > instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const
Targets can implement their own combinations for target-specific intrinsics.
std::optional< Value * > simplifyDemandedVectorEltsIntrinsic(InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3, std::function< void(Instruction *, unsigned, APInt, APInt &)> SimplifyAndSetOp) const
Can be used to implement target-specific instruction combining.
std::optional< Value * > simplifyDemandedUseBitsIntrinsic(InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known, bool &KnownBitsComputed) const
Can be used to implement target-specific instruction combining.
bool isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const
Query the target whether the specified address space cast from FromAS to ToAS is valid.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
const fltSemantics & getFltSemantics() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:270
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isStructTy() const
True if this is an instance of StructType.
Definition: Type.h:258
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:310
bool isScalableTy(SmallPtrSetImpl< const Type * > &Visited) const
Return true if this is a type whose size is a known multiple of vscale.
static IntegerType * getInt32Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:237
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:355
This class represents a cast unsigned integer to floating point.
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
op_range operands()
Definition: User.h:288
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition: User.cpp:21
op_iterator op_begin()
Definition: User.h:280
Value * getOperand(unsigned i) const
Definition: User.h:228
unsigned getNumOperands() const
Definition: User.h:250
op_iterator op_end()
Definition: User.h:282
bool isDroppable() const
A droppable user is a user for which uses can be dropped without affecting correctness and should be ...
Definition: User.cpp:115
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
const Value * stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, APInt &Offset) const
This is a wrapper around stripAndAccumulateConstantOffsets with the in-bounds requirement set to fals...
Definition: Value.h:740
bool hasOneUser() const
Return true if there is exactly one user of this value.
Definition: Value.cpp:157
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
iterator_range< user_iterator > users()
Definition: Value.h:421
bool hasNUses(unsigned N) const
Return true if this Value has exactly N uses.
Definition: Value.cpp:149
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:694
bool use_empty() const
Definition: Value.h:344
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
uint64_t getPointerDereferenceableBytes(const DataLayout &DL, bool &CanBeNull, bool &CanBeFreed) const
Returns the number of bytes known to be dereferenceable for the pointer value.
Definition: Value.cpp:852
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:383
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:202
constexpr bool isZero() const
Definition: TypeSize.h:156
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition: ilist_node.h:32
reverse_self_iterator getReverseIterator()
Definition: ilist_node.h:135
self_iterator getIterator()
Definition: ilist_node.h:132
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:661
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
bool isNoFPClassCompatibleType(Type *Ty)
Returns true if this is a type legal for the 'nofpclass' attribute.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition: Intrinsics.cpp:731
cst_pred_ty< is_all_ones > m_AllOnes()
Match an integer or vector with all bits set.
Definition: PatternMatch.h:524
class_match< PoisonValue > m_Poison()
Match an arbitrary poison constant.
Definition: PatternMatch.h:160
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
PtrAdd_match< PointerOpTy, OffsetOpTy > m_PtrAdd(const PointerOpTy &PointerOp, const OffsetOpTy &OffsetOp)
Matches GEP with i8 source element type.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
Definition: PatternMatch.h:100
CmpClass_match< LHS, RHS, FCmpInst > m_FCmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
Definition: PatternMatch.h:165
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
br_match m_UnconditionalBr(BasicBlock *&Succ)
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
Definition: PatternMatch.h:982
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
BinOpPred_match< LHS, RHS, is_idiv_op > m_IDiv(const LHS &L, const RHS &R)
Matches integer division operations.
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
Definition: PatternMatch.h:826
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:885
DisjointOr_match< LHS, RHS > m_DisjointOr(const LHS &L, const RHS &R)
constantexpr_match m_ConstantExpr()
Match a constant expression or a constant that contains a constant expression.
Definition: PatternMatch.h:186
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
cst_pred_ty< is_nonnegative > m_NonNegative()
Match an integer or vector of non-negative values.
Definition: PatternMatch.h:560
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
Definition: PatternMatch.h:168
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
apint_match m_APIntAllowPoison(const APInt *&Res)
Match APInt while allowing poison in splat vector constants.
Definition: PatternMatch.h:305
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
match_combine_and< class_match< Constant >, match_unless< constantexpr_match > > m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
Definition: PatternMatch.h:864
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
BinaryOp_match< LHS, RHS, Instruction::UDiv > m_UDiv(const LHS &L, const RHS &R)
brc_match< Cond_t, bind_ty< BasicBlock >, bind_ty< BasicBlock > > m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F)
match_combine_or< BinaryOp_match< LHS, RHS, Instruction::Add >, DisjointOr_match< LHS, RHS > > m_AddLike(const LHS &L, const RHS &R)
Match either "add" or "or disjoint".
CastInst_match< OpTy, UIToFPInst > m_UIToFP(const OpTy &Op)
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
match_combine_or< CastInst_match< OpTy, SExtInst >, NNegZExt_match< OpTy > > m_SExtLike(const OpTy &Op)
Match either "sext" or "zext nneg".
BinaryOp_match< LHS, RHS, Instruction::SDiv > m_SDiv(const LHS &L, const RHS &R)
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
Definition: PatternMatch.h:299
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
AnyBinaryOp_match< LHS, RHS, true > m_c_BinOp(const LHS &L, const RHS &R)
Matches a BinaryOperator with LHS and RHS in either order.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoSignedWrap > m_NSWAdd(const LHS &L, const RHS &R)
CastInst_match< OpTy, SIToFPInst > m_SIToFP(const OpTy &Op)
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
BinOpPred_match< LHS, RHS, is_shift_op > m_Shift(const LHS &L, const RHS &R)
Matches shift operations.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
cstfp_pred_ty< is_non_zero_fp > m_NonZeroFP()
Match a floating-point non-zero.
Definition: PatternMatch.h:791
m_Intrinsic_Ty< Opnd0 >::Ty m_VecReverse(const Opnd0 &Op0)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
match_combine_or< match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > >, match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > > > m_MaxOrMin(const LHS &L, const RHS &R)
auto m_Undef()
Match an arbitrary undef constant.
Definition: PatternMatch.h:152
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
Definition: PatternMatch.h:612
CastOperator_match< OpTy, Instruction::PtrToInt > m_PtrToInt(const OpTy &Op)
Matches PtrToInt.
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
Definition: PatternMatch.h:239
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
@ FalseVal
Definition: TGLexer.h:59
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
Intrinsic::ID getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID)
@ Offset
Definition: DWP.cpp:480
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition: STLExtras.h:854
void stable_sort(R &&Range)
Definition: STLExtras.h:2037
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1739
Value * simplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef< Value * > Indices, GEPNoWrapFlags NW, const SimplifyQuery &Q)
Given operands for a GetElementPtrInst, fold the result or return null.
bool succ_empty(const Instruction *I)
Definition: CFG.h:255
Value * simplifyFreezeInst(Value *Op, const SimplifyQuery &Q)
Given an operand for a Freeze, see if we can fold the result.
FunctionPass * createInstructionCombiningPass()
bool isSafeToSpeculativelyExecuteWithVariableReplaced(const Instruction *I)
Don't use information from its non-constant operands.
std::pair< unsigned, unsigned > removeAllNonTerminatorAndEHPadInstructions(BasicBlock *BB)
Remove all instructions from a basic block other than its terminator and any present EH pad instructi...
Definition: Local.cpp:2877
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition: STLExtras.h:2448
void salvageDebugInfoForDbgValues(Instruction &I, ArrayRef< DbgVariableIntrinsic * > Insns, ArrayRef< DbgVariableRecord * > DPInsns)
Implementation of salvageDebugInfo, applying only to instructions in Insns, rather than all debug use...
Definition: Local.cpp:2316
void findDbgUsers(SmallVectorImpl< DbgVariableIntrinsic * > &DbgInsts, Value *V, SmallVectorImpl< DbgVariableRecord * > *DbgVariableRecords=nullptr)
Finds the debug info intrinsics describing a value.
Definition: DebugInfo.cpp:162
void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition: Utils.cpp:1683
auto successors(const MachineBasicBlock *BB)
bool isRemovableAlloc(const CallBase *V, const TargetLibraryInfo *TLI)
Return true if this is a call to an allocation function that does not have side effects that we are r...
std::optional< StringRef > getAllocationFamily(const Value *I, const TargetLibraryInfo *TLI)
If a function is part of an allocation family (e.g.
Value * lowerObjectSizeCall(IntrinsicInst *ObjectSize, const DataLayout &DL, const TargetLibraryInfo *TLI, bool MustSucceed)
Try to turn a call to @llvm.objectsize into an integer value of the given Type.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Value * simplifyInstructionWithOperands(Instruction *I, ArrayRef< Value * > NewOps, const SimplifyQuery &Q)
Like simplifyInstruction but the operands of I are replaced with NewOps.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2115
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:657
gep_type_iterator gep_type_end(const User *GEP)
Value * getReallocatedOperand(const CallBase *CB)
If this is a call to a realloc function, return the reallocated operand.
bool isAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI)
Tests if a value is a call or invoke to a library function that allocates memory (either malloc,...
bool handleUnreachableTerminator(Instruction *I, SmallVectorImpl< Value * > &PoisonedValues)
If a terminator in an unreachable basic block has an operand of type Instruction, transform it into p...
Definition: Local.cpp:2859
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
Value * simplifyAddInst(Value *LHS, Value *RHS, bool IsNSW, bool IsNUW, const SimplifyQuery &Q)
Given operands for an Add, fold the result or return null.
Constant * ConstantFoldConstant(const Constant *C, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldConstant - Fold the constant using the specified DataLayout.
constexpr bool has_single_bit(T Value) noexcept
Definition: bit.h:146
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1746
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
Definition: Local.cpp:406
bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
Value * emitGEPOffset(IRBuilderBase *Builder, const DataLayout &DL, User *GEP, bool NoAssumptions=false)
Given a getelementptr instruction/constantexpr, emit the code necessary to compute the offset from th...
Definition: Local.cpp:22
constexpr unsigned MaxAnalysisRecursionDepth
Definition: ValueTracking.h:44
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:420
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1664
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
bool LowerDbgDeclare(Function &F)
Lowers llvm.dbg.declare intrinsics into appropriate set of llvm.dbg.value intrinsics.
Definition: Local.cpp:1990
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
void ConvertDebugDeclareToDebugValue(DbgVariableIntrinsic *DII, StoreInst *SI, DIBuilder &Builder)
Inserts a llvm.dbg.value intrinsic before a store to an alloca'd value that has an associated llvm....
Definition: Local.cpp:1731
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
bool canCreateUndefOrPoison(const Operator *Op, bool ConsiderFlagsAndMetadata=true)
canCreateUndefOrPoison returns true if Op can create undef or poison from non-undef & non-poison oper...
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Value * simplifyExtractValueInst(Value *Agg, ArrayRef< unsigned > Idxs, const SimplifyQuery &Q)
Given operands for an ExtractValueInst, fold the result or return null.
Constant * ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL)
Attempt to constant fold a binary operation with the specified operands.
bool replaceAllDbgUsesWith(Instruction &From, Value &To, Instruction &DomPoint, DominatorTree &DT)
Point debug users of From to To or salvage them.
Definition: Local.cpp:2787
bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
constexpr int PoisonMaskElem
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition: STLExtras.h:336
Value * simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a BinaryOperator, fold the result or return null.
@ Or
Bitwise or logical OR of integers.
DWARFExpression::Operation Op
Constant * ConstantFoldInstruction(Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldInstruction - Try to constant fold the specified instruction.
bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
Value * getFreedOperand(const CallBase *CB, const TargetLibraryInfo *TLI)
If this if a call to a free function, return the freed operand.
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:217
bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
gep_type_iterator gep_type_begin(const User *GEP)
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1903
bool equal(L &&LRange, R &&RRange)
Wrapper function around std::equal to detect if pair-wise elements between two ranges are the same.
Definition: STLExtras.h:2067
bool isKnownNonNegative(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the give value is known to be non-negative.
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
void initializeInstCombine(PassRegistry &)
Initialize all passes linked into the InstCombine library.
void initializeInstructionCombiningPassPass(PassRegistry &)
Constant * ConstantFoldBinaryInstruction(unsigned Opcode, Constant *V1, Constant *V2)
std::optional< bool > isImpliedCondition(const Value *LHS, const Value *RHS, const DataLayout &DL, bool LHSIsTrue=true, unsigned Depth=0)
Return true if RHS is known to be implied true by LHS.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
static unsigned int semanticsPrecision(const fltSemantics &)
Definition: APFloat.cpp:317
unsigned countMinLeadingOnes() const
Returns the minimum number of leading one bits.
Definition: KnownBits.h:243
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:43
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:240
A CRTP mix-in to automatically provide informational APIs needed for passes.
Definition: PassManager.h:69
SimplifyQuery getWithInstruction(const Instruction *I) const
SimplifyQuery getWithoutUndef() const