LLVM 22.0.0git
InstructionCombining.cpp
Go to the documentation of this file.
1//===- InstructionCombining.cpp - Combine multiple instructions -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// InstructionCombining - Combine instructions to form fewer, simple
10// instructions. This pass does not modify the CFG. This pass is where
11// algebraic simplification happens.
12//
13// This pass combines things like:
14// %Y = add i32 %X, 1
15// %Z = add i32 %Y, 1
16// into:
17// %Z = add i32 %X, 2
18//
19// This is a simple worklist driven algorithm.
20//
21// This pass guarantees that the following canonicalizations are performed on
22// the program:
23// 1. If a binary operator has a constant operand, it is moved to the RHS
24// 2. Bitwise operators with constant operands are always grouped so that
25// shifts are performed first, then or's, then and's, then xor's.
26// 3. Compare instructions are converted from <,>,<=,>= to ==,!= if possible
27// 4. All cmp instructions on boolean values are replaced with logical ops
28// 5. add X, X is represented as (X*2) => (X << 1)
29// 6. Multiplies with a power-of-two constant argument are transformed into
30// shifts.
31// ... etc.
32//
33//===----------------------------------------------------------------------===//
34
35#include "InstCombineInternal.h"
36#include "llvm/ADT/APFloat.h"
37#include "llvm/ADT/APInt.h"
38#include "llvm/ADT/ArrayRef.h"
39#include "llvm/ADT/DenseMap.h"
42#include "llvm/ADT/Statistic.h"
47#include "llvm/Analysis/CFG.h"
62#include "llvm/IR/BasicBlock.h"
63#include "llvm/IR/CFG.h"
64#include "llvm/IR/Constant.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DIBuilder.h"
67#include "llvm/IR/DataLayout.h"
68#include "llvm/IR/DebugInfo.h"
70#include "llvm/IR/Dominators.h"
72#include "llvm/IR/Function.h"
74#include "llvm/IR/IRBuilder.h"
75#include "llvm/IR/InstrTypes.h"
76#include "llvm/IR/Instruction.h"
79#include "llvm/IR/Intrinsics.h"
80#include "llvm/IR/Metadata.h"
81#include "llvm/IR/Operator.h"
82#include "llvm/IR/PassManager.h"
84#include "llvm/IR/Type.h"
85#include "llvm/IR/Use.h"
86#include "llvm/IR/User.h"
87#include "llvm/IR/Value.h"
88#include "llvm/IR/ValueHandle.h"
93#include "llvm/Support/Debug.h"
102#include <algorithm>
103#include <cassert>
104#include <cstdint>
105#include <memory>
106#include <optional>
107#include <string>
108#include <utility>
109
110#define DEBUG_TYPE "instcombine"
112#include <optional>
113
114using namespace llvm;
115using namespace llvm::PatternMatch;
116
117STATISTIC(NumWorklistIterations,
118 "Number of instruction combining iterations performed");
119STATISTIC(NumOneIteration, "Number of functions with one iteration");
120STATISTIC(NumTwoIterations, "Number of functions with two iterations");
121STATISTIC(NumThreeIterations, "Number of functions with three iterations");
122STATISTIC(NumFourOrMoreIterations,
123 "Number of functions with four or more iterations");
124
125STATISTIC(NumCombined , "Number of insts combined");
126STATISTIC(NumConstProp, "Number of constant folds");
127STATISTIC(NumDeadInst , "Number of dead inst eliminated");
128STATISTIC(NumSunkInst , "Number of instructions sunk");
129STATISTIC(NumExpand, "Number of expansions");
130STATISTIC(NumFactor , "Number of factorizations");
131STATISTIC(NumReassoc , "Number of reassociations");
132DEBUG_COUNTER(VisitCounter, "instcombine-visit",
133 "Controls which instructions are visited");
134
135static cl::opt<bool> EnableCodeSinking("instcombine-code-sinking",
136 cl::desc("Enable code sinking"),
137 cl::init(true));
138
140 "instcombine-max-sink-users", cl::init(32),
141 cl::desc("Maximum number of undroppable users for instruction sinking"));
142
144MaxArraySize("instcombine-maxarray-size", cl::init(1024),
145 cl::desc("Maximum array size considered when doing a combine"));
146
147namespace llvm {
149} // end namespace llvm
150
151// FIXME: Remove this flag when it is no longer necessary to convert
152// llvm.dbg.declare to avoid inaccurate debug info. Setting this to false
153// increases variable availability at the cost of accuracy. Variables that
154// cannot be promoted by mem2reg or SROA will be described as living in memory
155// for their entire lifetime. However, passes like DSE and instcombine can
156// delete stores to the alloca, leading to misleading and inaccurate debug
157// information. This flag can be removed when those passes are fixed.
158static cl::opt<unsigned> ShouldLowerDbgDeclare("instcombine-lower-dbg-declare",
159 cl::Hidden, cl::init(true));
160
161std::optional<Instruction *>
163 // Handle target specific intrinsics
164 if (II.getCalledFunction()->isTargetIntrinsic()) {
165 return TTIForTargetIntrinsicsOnly.instCombineIntrinsic(*this, II);
166 }
167 return std::nullopt;
168}
169
171 IntrinsicInst &II, APInt DemandedMask, KnownBits &Known,
172 bool &KnownBitsComputed) {
173 // Handle target specific intrinsics
174 if (II.getCalledFunction()->isTargetIntrinsic()) {
175 return TTIForTargetIntrinsicsOnly.simplifyDemandedUseBitsIntrinsic(
176 *this, II, DemandedMask, Known, KnownBitsComputed);
177 }
178 return std::nullopt;
179}
180
182 IntrinsicInst &II, APInt DemandedElts, APInt &PoisonElts,
183 APInt &PoisonElts2, APInt &PoisonElts3,
184 std::function<void(Instruction *, unsigned, APInt, APInt &)>
185 SimplifyAndSetOp) {
186 // Handle target specific intrinsics
187 if (II.getCalledFunction()->isTargetIntrinsic()) {
188 return TTIForTargetIntrinsicsOnly.simplifyDemandedVectorEltsIntrinsic(
189 *this, II, DemandedElts, PoisonElts, PoisonElts2, PoisonElts3,
190 SimplifyAndSetOp);
191 }
192 return std::nullopt;
193}
194
195bool InstCombiner::isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const {
196 // Approved exception for TTI use: This queries a legality property of the
197 // target, not an profitability heuristic. Ideally this should be part of
198 // DataLayout instead.
199 return TTIForTargetIntrinsicsOnly.isValidAddrSpaceCast(FromAS, ToAS);
200}
201
202Value *InstCombinerImpl::EmitGEPOffset(GEPOperator *GEP, bool RewriteGEP) {
203 if (!RewriteGEP)
204 return llvm::emitGEPOffset(&Builder, DL, GEP);
205
206 IRBuilderBase::InsertPointGuard Guard(Builder);
207 auto *Inst = dyn_cast<Instruction>(GEP);
208 if (Inst)
209 Builder.SetInsertPoint(Inst);
210
211 Value *Offset = EmitGEPOffset(GEP);
212 // Rewrite non-trivial GEPs to avoid duplicating the offset arithmetic.
213 if (Inst && !GEP->hasAllConstantIndices() &&
214 !GEP->getSourceElementType()->isIntegerTy(8)) {
216 *Inst, Builder.CreateGEP(Builder.getInt8Ty(), GEP->getPointerOperand(),
217 Offset, "", GEP->getNoWrapFlags()));
219 }
220 return Offset;
221}
222
223Value *InstCombinerImpl::EmitGEPOffsets(ArrayRef<GEPOperator *> GEPs,
224 GEPNoWrapFlags NW, Type *IdxTy,
225 bool RewriteGEPs) {
226 auto Add = [&](Value *Sum, Value *Offset) -> Value * {
227 if (Sum)
228 return Builder.CreateAdd(Sum, Offset, "", NW.hasNoUnsignedWrap(),
229 NW.isInBounds());
230 else
231 return Offset;
232 };
233
234 Value *Sum = nullptr;
235 Value *OneUseSum = nullptr;
236 Value *OneUseBase = nullptr;
237 GEPNoWrapFlags OneUseFlags = GEPNoWrapFlags::all();
238 for (GEPOperator *GEP : reverse(GEPs)) {
239 Value *Offset;
240 {
241 // Expand the offset at the point of the previous GEP to enable rewriting.
242 // However, use the original insertion point for calculating Sum.
243 IRBuilderBase::InsertPointGuard Guard(Builder);
244 auto *Inst = dyn_cast<Instruction>(GEP);
245 if (RewriteGEPs && Inst)
246 Builder.SetInsertPoint(Inst);
247
249 if (Offset->getType() != IdxTy)
250 Offset = Builder.CreateVectorSplat(
251 cast<VectorType>(IdxTy)->getElementCount(), Offset);
252 if (GEP->hasOneUse()) {
253 // Offsets of one-use GEPs will be merged into the next multi-use GEP.
254 OneUseSum = Add(OneUseSum, Offset);
255 OneUseFlags = OneUseFlags.intersectForOffsetAdd(GEP->getNoWrapFlags());
256 if (!OneUseBase)
257 OneUseBase = GEP->getPointerOperand();
258 continue;
259 }
260
261 if (OneUseSum)
262 Offset = Add(OneUseSum, Offset);
263
264 // Rewrite the GEP to reuse the computed offset. This also includes
265 // offsets from preceding one-use GEPs.
266 if (RewriteGEPs && Inst &&
267 !(GEP->getSourceElementType()->isIntegerTy(8) &&
268 GEP->getOperand(1) == Offset)) {
270 *Inst,
271 Builder.CreatePtrAdd(
272 OneUseBase ? OneUseBase : GEP->getPointerOperand(), Offset, "",
273 OneUseFlags.intersectForOffsetAdd(GEP->getNoWrapFlags())));
275 }
276 }
277
278 Sum = Add(Sum, Offset);
279 OneUseSum = OneUseBase = nullptr;
280 OneUseFlags = GEPNoWrapFlags::all();
281 }
282 if (OneUseSum)
283 Sum = Add(Sum, OneUseSum);
284 if (!Sum)
285 return Constant::getNullValue(IdxTy);
286 return Sum;
287}
288
289/// Legal integers and common types are considered desirable. This is used to
290/// avoid creating instructions with types that may not be supported well by the
291/// the backend.
292/// NOTE: This treats i8, i16 and i32 specially because they are common
293/// types in frontend languages.
294bool InstCombinerImpl::isDesirableIntType(unsigned BitWidth) const {
295 switch (BitWidth) {
296 case 8:
297 case 16:
298 case 32:
299 return true;
300 default:
301 return DL.isLegalInteger(BitWidth);
302 }
303}
304
305/// Return true if it is desirable to convert an integer computation from a
306/// given bit width to a new bit width.
307/// We don't want to convert from a legal or desirable type (like i8) to an
308/// illegal type or from a smaller to a larger illegal type. A width of '1'
309/// is always treated as a desirable type because i1 is a fundamental type in
310/// IR, and there are many specialized optimizations for i1 types.
311/// Common/desirable widths are equally treated as legal to convert to, in
312/// order to open up more combining opportunities.
313bool InstCombinerImpl::shouldChangeType(unsigned FromWidth,
314 unsigned ToWidth) const {
315 bool FromLegal = FromWidth == 1 || DL.isLegalInteger(FromWidth);
316 bool ToLegal = ToWidth == 1 || DL.isLegalInteger(ToWidth);
317
318 // Convert to desirable widths even if they are not legal types.
319 // Only shrink types, to prevent infinite loops.
320 if (ToWidth < FromWidth && isDesirableIntType(ToWidth))
321 return true;
322
323 // If this is a legal or desiable integer from type, and the result would be
324 // an illegal type, don't do the transformation.
325 if ((FromLegal || isDesirableIntType(FromWidth)) && !ToLegal)
326 return false;
327
328 // Otherwise, if both are illegal, do not increase the size of the result. We
329 // do allow things like i160 -> i64, but not i64 -> i160.
330 if (!FromLegal && !ToLegal && ToWidth > FromWidth)
331 return false;
332
333 return true;
334}
335
336/// Return true if it is desirable to convert a computation from 'From' to 'To'.
337/// We don't want to convert from a legal to an illegal type or from a smaller
338/// to a larger illegal type. i1 is always treated as a legal type because it is
339/// a fundamental type in IR, and there are many specialized optimizations for
340/// i1 types.
341bool InstCombinerImpl::shouldChangeType(Type *From, Type *To) const {
342 // TODO: This could be extended to allow vectors. Datalayout changes might be
343 // needed to properly support that.
344 if (!From->isIntegerTy() || !To->isIntegerTy())
345 return false;
346
347 unsigned FromWidth = From->getPrimitiveSizeInBits();
348 unsigned ToWidth = To->getPrimitiveSizeInBits();
349 return shouldChangeType(FromWidth, ToWidth);
350}
351
352// Return true, if No Signed Wrap should be maintained for I.
353// The No Signed Wrap flag can be kept if the operation "B (I.getOpcode) C",
354// where both B and C should be ConstantInts, results in a constant that does
355// not overflow. This function only handles the Add/Sub/Mul opcodes. For
356// all other opcodes, the function conservatively returns false.
359 if (!OBO || !OBO->hasNoSignedWrap())
360 return false;
361
362 const APInt *BVal, *CVal;
363 if (!match(B, m_APInt(BVal)) || !match(C, m_APInt(CVal)))
364 return false;
365
366 // We reason about Add/Sub/Mul Only.
367 bool Overflow = false;
368 switch (I.getOpcode()) {
369 case Instruction::Add:
370 (void)BVal->sadd_ov(*CVal, Overflow);
371 break;
372 case Instruction::Sub:
373 (void)BVal->ssub_ov(*CVal, Overflow);
374 break;
375 case Instruction::Mul:
376 (void)BVal->smul_ov(*CVal, Overflow);
377 break;
378 default:
379 // Conservatively return false for other opcodes.
380 return false;
381 }
382 return !Overflow;
383}
384
387 return OBO && OBO->hasNoUnsignedWrap();
388}
389
392 return OBO && OBO->hasNoSignedWrap();
393}
394
395/// Conservatively clears subclassOptionalData after a reassociation or
396/// commutation. We preserve fast-math flags when applicable as they can be
397/// preserved.
400 if (!FPMO) {
401 I.clearSubclassOptionalData();
402 return;
403 }
404
405 FastMathFlags FMF = I.getFastMathFlags();
406 I.clearSubclassOptionalData();
407 I.setFastMathFlags(FMF);
408}
409
410/// Combine constant operands of associative operations either before or after a
411/// cast to eliminate one of the associative operations:
412/// (op (cast (op X, C2)), C1) --> (cast (op X, op (C1, C2)))
413/// (op (cast (op X, C2)), C1) --> (op (cast X), op (C1, C2))
415 InstCombinerImpl &IC) {
416 auto *Cast = dyn_cast<CastInst>(BinOp1->getOperand(0));
417 if (!Cast || !Cast->hasOneUse())
418 return false;
419
420 // TODO: Enhance logic for other casts and remove this check.
421 auto CastOpcode = Cast->getOpcode();
422 if (CastOpcode != Instruction::ZExt)
423 return false;
424
425 // TODO: Enhance logic for other BinOps and remove this check.
426 if (!BinOp1->isBitwiseLogicOp())
427 return false;
428
429 auto AssocOpcode = BinOp1->getOpcode();
430 auto *BinOp2 = dyn_cast<BinaryOperator>(Cast->getOperand(0));
431 if (!BinOp2 || !BinOp2->hasOneUse() || BinOp2->getOpcode() != AssocOpcode)
432 return false;
433
434 Constant *C1, *C2;
435 if (!match(BinOp1->getOperand(1), m_Constant(C1)) ||
436 !match(BinOp2->getOperand(1), m_Constant(C2)))
437 return false;
438
439 // TODO: This assumes a zext cast.
440 // Eg, if it was a trunc, we'd cast C1 to the source type because casting C2
441 // to the destination type might lose bits.
442
443 // Fold the constants together in the destination type:
444 // (op (cast (op X, C2)), C1) --> (op (cast X), FoldedC)
445 const DataLayout &DL = IC.getDataLayout();
446 Type *DestTy = C1->getType();
447 Constant *CastC2 = ConstantFoldCastOperand(CastOpcode, C2, DestTy, DL);
448 if (!CastC2)
449 return false;
450 Constant *FoldedC = ConstantFoldBinaryOpOperands(AssocOpcode, C1, CastC2, DL);
451 if (!FoldedC)
452 return false;
453
454 IC.replaceOperand(*Cast, 0, BinOp2->getOperand(0));
455 IC.replaceOperand(*BinOp1, 1, FoldedC);
457 Cast->dropPoisonGeneratingFlags();
458 return true;
459}
460
461// Simplifies IntToPtr/PtrToInt RoundTrip Cast.
462// inttoptr ( ptrtoint (x) ) --> x
463Value *InstCombinerImpl::simplifyIntToPtrRoundTripCast(Value *Val) {
464 auto *IntToPtr = dyn_cast<IntToPtrInst>(Val);
465 if (IntToPtr && DL.getTypeSizeInBits(IntToPtr->getDestTy()) ==
466 DL.getTypeSizeInBits(IntToPtr->getSrcTy())) {
467 auto *PtrToInt = dyn_cast<PtrToIntInst>(IntToPtr->getOperand(0));
468 Type *CastTy = IntToPtr->getDestTy();
469 if (PtrToInt &&
470 CastTy->getPointerAddressSpace() ==
471 PtrToInt->getSrcTy()->getPointerAddressSpace() &&
472 DL.getTypeSizeInBits(PtrToInt->getSrcTy()) ==
473 DL.getTypeSizeInBits(PtrToInt->getDestTy()))
474 return PtrToInt->getOperand(0);
475 }
476 return nullptr;
477}
478
479/// This performs a few simplifications for operators that are associative or
480/// commutative:
481///
482/// Commutative operators:
483///
484/// 1. Order operands such that they are listed from right (least complex) to
485/// left (most complex). This puts constants before unary operators before
486/// binary operators.
487///
488/// Associative operators:
489///
490/// 2. Transform: "(A op B) op C" ==> "A op (B op C)" if "B op C" simplifies.
491/// 3. Transform: "A op (B op C)" ==> "(A op B) op C" if "A op B" simplifies.
492///
493/// Associative and commutative operators:
494///
495/// 4. Transform: "(A op B) op C" ==> "(C op A) op B" if "C op A" simplifies.
496/// 5. Transform: "A op (B op C)" ==> "B op (C op A)" if "C op A" simplifies.
497/// 6. Transform: "(A op C1) op (B op C2)" ==> "(A op B) op (C1 op C2)"
498/// if C1 and C2 are constants.
500 Instruction::BinaryOps Opcode = I.getOpcode();
501 bool Changed = false;
502
503 do {
504 // Order operands such that they are listed from right (least complex) to
505 // left (most complex). This puts constants before unary operators before
506 // binary operators.
507 if (I.isCommutative() && getComplexity(I.getOperand(0)) <
508 getComplexity(I.getOperand(1)))
509 Changed = !I.swapOperands();
510
511 if (I.isCommutative()) {
512 if (auto Pair = matchSymmetricPair(I.getOperand(0), I.getOperand(1))) {
513 replaceOperand(I, 0, Pair->first);
514 replaceOperand(I, 1, Pair->second);
515 Changed = true;
516 }
517 }
518
519 BinaryOperator *Op0 = dyn_cast<BinaryOperator>(I.getOperand(0));
520 BinaryOperator *Op1 = dyn_cast<BinaryOperator>(I.getOperand(1));
521
522 if (I.isAssociative()) {
523 // Transform: "(A op B) op C" ==> "A op (B op C)" if "B op C" simplifies.
524 if (Op0 && Op0->getOpcode() == Opcode) {
525 Value *A = Op0->getOperand(0);
526 Value *B = Op0->getOperand(1);
527 Value *C = I.getOperand(1);
528
529 // Does "B op C" simplify?
530 if (Value *V = simplifyBinOp(Opcode, B, C, SQ.getWithInstruction(&I))) {
531 // It simplifies to V. Form "A op V".
532 replaceOperand(I, 0, A);
533 replaceOperand(I, 1, V);
534 bool IsNUW = hasNoUnsignedWrap(I) && hasNoUnsignedWrap(*Op0);
535 bool IsNSW = maintainNoSignedWrap(I, B, C) && hasNoSignedWrap(*Op0);
536
537 // Conservatively clear all optional flags since they may not be
538 // preserved by the reassociation. Reset nsw/nuw based on the above
539 // analysis.
541
542 // Note: this is only valid because SimplifyBinOp doesn't look at
543 // the operands to Op0.
544 if (IsNUW)
545 I.setHasNoUnsignedWrap(true);
546
547 if (IsNSW)
548 I.setHasNoSignedWrap(true);
549
550 Changed = true;
551 ++NumReassoc;
552 continue;
553 }
554 }
555
556 // Transform: "A op (B op C)" ==> "(A op B) op C" if "A op B" simplifies.
557 if (Op1 && Op1->getOpcode() == Opcode) {
558 Value *A = I.getOperand(0);
559 Value *B = Op1->getOperand(0);
560 Value *C = Op1->getOperand(1);
561
562 // Does "A op B" simplify?
563 if (Value *V = simplifyBinOp(Opcode, A, B, SQ.getWithInstruction(&I))) {
564 // It simplifies to V. Form "V op C".
565 replaceOperand(I, 0, V);
566 replaceOperand(I, 1, C);
567 // Conservatively clear the optional flags, since they may not be
568 // preserved by the reassociation.
570 Changed = true;
571 ++NumReassoc;
572 continue;
573 }
574 }
575 }
576
577 if (I.isAssociative() && I.isCommutative()) {
578 if (simplifyAssocCastAssoc(&I, *this)) {
579 Changed = true;
580 ++NumReassoc;
581 continue;
582 }
583
584 // Transform: "(A op B) op C" ==> "(C op A) op B" if "C op A" simplifies.
585 if (Op0 && Op0->getOpcode() == Opcode) {
586 Value *A = Op0->getOperand(0);
587 Value *B = Op0->getOperand(1);
588 Value *C = I.getOperand(1);
589
590 // Does "C op A" simplify?
591 if (Value *V = simplifyBinOp(Opcode, C, A, SQ.getWithInstruction(&I))) {
592 // It simplifies to V. Form "V op B".
593 replaceOperand(I, 0, V);
594 replaceOperand(I, 1, B);
595 // Conservatively clear the optional flags, since they may not be
596 // preserved by the reassociation.
598 Changed = true;
599 ++NumReassoc;
600 continue;
601 }
602 }
603
604 // Transform: "A op (B op C)" ==> "B op (C op A)" if "C op A" simplifies.
605 if (Op1 && Op1->getOpcode() == Opcode) {
606 Value *A = I.getOperand(0);
607 Value *B = Op1->getOperand(0);
608 Value *C = Op1->getOperand(1);
609
610 // Does "C op A" simplify?
611 if (Value *V = simplifyBinOp(Opcode, C, A, SQ.getWithInstruction(&I))) {
612 // It simplifies to V. Form "B op V".
613 replaceOperand(I, 0, B);
614 replaceOperand(I, 1, V);
615 // Conservatively clear the optional flags, since they may not be
616 // preserved by the reassociation.
618 Changed = true;
619 ++NumReassoc;
620 continue;
621 }
622 }
623
624 // Transform: "(A op C1) op (B op C2)" ==> "(A op B) op (C1 op C2)"
625 // if C1 and C2 are constants.
626 Value *A, *B;
627 Constant *C1, *C2, *CRes;
628 if (Op0 && Op1 &&
629 Op0->getOpcode() == Opcode && Op1->getOpcode() == Opcode &&
630 match(Op0, m_OneUse(m_BinOp(m_Value(A), m_Constant(C1)))) &&
631 match(Op1, m_OneUse(m_BinOp(m_Value(B), m_Constant(C2)))) &&
632 (CRes = ConstantFoldBinaryOpOperands(Opcode, C1, C2, DL))) {
633 bool IsNUW = hasNoUnsignedWrap(I) &&
634 hasNoUnsignedWrap(*Op0) &&
635 hasNoUnsignedWrap(*Op1);
636 BinaryOperator *NewBO = (IsNUW && Opcode == Instruction::Add) ?
637 BinaryOperator::CreateNUW(Opcode, A, B) :
638 BinaryOperator::Create(Opcode, A, B);
639
640 if (isa<FPMathOperator>(NewBO)) {
641 FastMathFlags Flags = I.getFastMathFlags() &
642 Op0->getFastMathFlags() &
643 Op1->getFastMathFlags();
644 NewBO->setFastMathFlags(Flags);
645 }
646 InsertNewInstWith(NewBO, I.getIterator());
647 NewBO->takeName(Op1);
648 replaceOperand(I, 0, NewBO);
649 replaceOperand(I, 1, CRes);
650 // Conservatively clear the optional flags, since they may not be
651 // preserved by the reassociation.
653 if (IsNUW)
654 I.setHasNoUnsignedWrap(true);
655
656 Changed = true;
657 continue;
658 }
659 }
660
661 // No further simplifications.
662 return Changed;
663 } while (true);
664}
665
666/// Return whether "X LOp (Y ROp Z)" is always equal to
667/// "(X LOp Y) ROp (X LOp Z)".
670 // X & (Y | Z) <--> (X & Y) | (X & Z)
671 // X & (Y ^ Z) <--> (X & Y) ^ (X & Z)
672 if (LOp == Instruction::And)
673 return ROp == Instruction::Or || ROp == Instruction::Xor;
674
675 // X | (Y & Z) <--> (X | Y) & (X | Z)
676 if (LOp == Instruction::Or)
677 return ROp == Instruction::And;
678
679 // X * (Y + Z) <--> (X * Y) + (X * Z)
680 // X * (Y - Z) <--> (X * Y) - (X * Z)
681 if (LOp == Instruction::Mul)
682 return ROp == Instruction::Add || ROp == Instruction::Sub;
683
684 return false;
685}
686
687/// Return whether "(X LOp Y) ROp Z" is always equal to
688/// "(X ROp Z) LOp (Y ROp Z)".
692 return leftDistributesOverRight(ROp, LOp);
693
694 // (X {&|^} Y) >> Z <--> (X >> Z) {&|^} (Y >> Z) for all shifts.
696
697 // TODO: It would be nice to handle division, aka "(X + Y)/Z = X/Z + Y/Z",
698 // but this requires knowing that the addition does not overflow and other
699 // such subtleties.
700}
701
702/// This function returns identity value for given opcode, which can be used to
703/// factor patterns like (X * 2) + X ==> (X * 2) + (X * 1) ==> X * (2 + 1).
705 if (isa<Constant>(V))
706 return nullptr;
707
708 return ConstantExpr::getBinOpIdentity(Opcode, V->getType());
709}
710
711/// This function predicates factorization using distributive laws. By default,
712/// it just returns the 'Op' inputs. But for special-cases like
713/// 'add(shl(X, 5), ...)', this function will have TopOpcode == Instruction::Add
714/// and Op = shl(X, 5). The 'shl' is treated as the more general 'mul X, 32' to
715/// allow more factorization opportunities.
718 Value *&LHS, Value *&RHS, BinaryOperator *OtherOp) {
719 assert(Op && "Expected a binary operator");
720 LHS = Op->getOperand(0);
721 RHS = Op->getOperand(1);
722 if (TopOpcode == Instruction::Add || TopOpcode == Instruction::Sub) {
723 Constant *C;
724 if (match(Op, m_Shl(m_Value(), m_ImmConstant(C)))) {
725 // X << C --> X * (1 << C)
727 Instruction::Shl, ConstantInt::get(Op->getType(), 1), C);
728 assert(RHS && "Constant folding of immediate constants failed");
729 return Instruction::Mul;
730 }
731 // TODO: We can add other conversions e.g. shr => div etc.
732 }
733 if (Instruction::isBitwiseLogicOp(TopOpcode)) {
734 if (OtherOp && OtherOp->getOpcode() == Instruction::AShr &&
736 // lshr nneg C, X --> ashr nneg C, X
737 return Instruction::AShr;
738 }
739 }
740 return Op->getOpcode();
741}
742
743/// This tries to simplify binary operations by factorizing out common terms
744/// (e. g. "(A*B)+(A*C)" -> "A*(B+C)").
747 Instruction::BinaryOps InnerOpcode, Value *A,
748 Value *B, Value *C, Value *D) {
749 assert(A && B && C && D && "All values must be provided");
750
751 Value *V = nullptr;
752 Value *RetVal = nullptr;
753 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
754 Instruction::BinaryOps TopLevelOpcode = I.getOpcode();
755
756 // Does "X op' Y" always equal "Y op' X"?
757 bool InnerCommutative = Instruction::isCommutative(InnerOpcode);
758
759 // Does "X op' (Y op Z)" always equal "(X op' Y) op (X op' Z)"?
760 if (leftDistributesOverRight(InnerOpcode, TopLevelOpcode)) {
761 // Does the instruction have the form "(A op' B) op (A op' D)" or, in the
762 // commutative case, "(A op' B) op (C op' A)"?
763 if (A == C || (InnerCommutative && A == D)) {
764 if (A != C)
765 std::swap(C, D);
766 // Consider forming "A op' (B op D)".
767 // If "B op D" simplifies then it can be formed with no cost.
768 V = simplifyBinOp(TopLevelOpcode, B, D, SQ.getWithInstruction(&I));
769
770 // If "B op D" doesn't simplify then only go on if one of the existing
771 // operations "A op' B" and "C op' D" will be zapped as no longer used.
772 if (!V && (LHS->hasOneUse() || RHS->hasOneUse()))
773 V = Builder.CreateBinOp(TopLevelOpcode, B, D, RHS->getName());
774 if (V)
775 RetVal = Builder.CreateBinOp(InnerOpcode, A, V);
776 }
777 }
778
779 // Does "(X op Y) op' Z" always equal "(X op' Z) op (Y op' Z)"?
780 if (!RetVal && rightDistributesOverLeft(TopLevelOpcode, InnerOpcode)) {
781 // Does the instruction have the form "(A op' B) op (C op' B)" or, in the
782 // commutative case, "(A op' B) op (B op' D)"?
783 if (B == D || (InnerCommutative && B == C)) {
784 if (B != D)
785 std::swap(C, D);
786 // Consider forming "(A op C) op' B".
787 // If "A op C" simplifies then it can be formed with no cost.
788 V = simplifyBinOp(TopLevelOpcode, A, C, SQ.getWithInstruction(&I));
789
790 // If "A op C" doesn't simplify then only go on if one of the existing
791 // operations "A op' B" and "C op' D" will be zapped as no longer used.
792 if (!V && (LHS->hasOneUse() || RHS->hasOneUse()))
793 V = Builder.CreateBinOp(TopLevelOpcode, A, C, LHS->getName());
794 if (V)
795 RetVal = Builder.CreateBinOp(InnerOpcode, V, B);
796 }
797 }
798
799 if (!RetVal)
800 return nullptr;
801
802 ++NumFactor;
803 RetVal->takeName(&I);
804
805 // Try to add no-overflow flags to the final value.
806 if (isa<BinaryOperator>(RetVal)) {
807 bool HasNSW = false;
808 bool HasNUW = false;
810 HasNSW = I.hasNoSignedWrap();
811 HasNUW = I.hasNoUnsignedWrap();
812 }
813 if (auto *LOBO = dyn_cast<OverflowingBinaryOperator>(LHS)) {
814 HasNSW &= LOBO->hasNoSignedWrap();
815 HasNUW &= LOBO->hasNoUnsignedWrap();
816 }
817
818 if (auto *ROBO = dyn_cast<OverflowingBinaryOperator>(RHS)) {
819 HasNSW &= ROBO->hasNoSignedWrap();
820 HasNUW &= ROBO->hasNoUnsignedWrap();
821 }
822
823 if (TopLevelOpcode == Instruction::Add && InnerOpcode == Instruction::Mul) {
824 // We can propagate 'nsw' if we know that
825 // %Y = mul nsw i16 %X, C
826 // %Z = add nsw i16 %Y, %X
827 // =>
828 // %Z = mul nsw i16 %X, C+1
829 //
830 // iff C+1 isn't INT_MIN
831 const APInt *CInt;
832 if (match(V, m_APInt(CInt)) && !CInt->isMinSignedValue())
833 cast<Instruction>(RetVal)->setHasNoSignedWrap(HasNSW);
834
835 // nuw can be propagated with any constant or nuw value.
836 cast<Instruction>(RetVal)->setHasNoUnsignedWrap(HasNUW);
837 }
838 }
839 return RetVal;
840}
841
842// If `I` has one Const operand and the other matches `(ctpop (not x))`,
843// replace `(ctpop (not x))` with `(sub nuw nsw BitWidth(x), (ctpop x))`.
844// This is only useful is the new subtract can fold so we only handle the
845// following cases:
846// 1) (add/sub/disjoint_or C, (ctpop (not x))
847// -> (add/sub/disjoint_or C', (ctpop x))
848// 1) (cmp pred C, (ctpop (not x))
849// -> (cmp pred C', (ctpop x))
851 unsigned Opc = I->getOpcode();
852 unsigned ConstIdx = 1;
853 switch (Opc) {
854 default:
855 return nullptr;
856 // (ctpop (not x)) <-> (sub nuw nsw BitWidth(x) - (ctpop x))
857 // We can fold the BitWidth(x) with add/sub/icmp as long the other operand
858 // is constant.
859 case Instruction::Sub:
860 ConstIdx = 0;
861 break;
862 case Instruction::ICmp:
863 // Signed predicates aren't correct in some edge cases like for i2 types, as
864 // well since (ctpop x) is known [0, log2(BitWidth(x))] almost all signed
865 // comparisons against it are simplfied to unsigned.
866 if (cast<ICmpInst>(I)->isSigned())
867 return nullptr;
868 break;
869 case Instruction::Or:
870 if (!match(I, m_DisjointOr(m_Value(), m_Value())))
871 return nullptr;
872 [[fallthrough]];
873 case Instruction::Add:
874 break;
875 }
876
877 Value *Op;
878 // Find ctpop.
879 if (!match(I->getOperand(1 - ConstIdx),
881 return nullptr;
882
883 Constant *C;
884 // Check other operand is ImmConstant.
885 if (!match(I->getOperand(ConstIdx), m_ImmConstant(C)))
886 return nullptr;
887
888 Type *Ty = Op->getType();
889 Constant *BitWidthC = ConstantInt::get(Ty, Ty->getScalarSizeInBits());
890 // Need extra check for icmp. Note if this check is true, it generally means
891 // the icmp will simplify to true/false.
892 if (Opc == Instruction::ICmp && !cast<ICmpInst>(I)->isEquality()) {
893 Constant *Cmp =
895 if (!Cmp || !Cmp->isZeroValue())
896 return nullptr;
897 }
898
899 // Check we can invert `(not x)` for free.
900 bool Consumes = false;
901 if (!isFreeToInvert(Op, Op->hasOneUse(), Consumes) || !Consumes)
902 return nullptr;
903 Value *NotOp = getFreelyInverted(Op, Op->hasOneUse(), &Builder);
904 assert(NotOp != nullptr &&
905 "Desync between isFreeToInvert and getFreelyInverted");
906
907 Value *CtpopOfNotOp = Builder.CreateIntrinsic(Ty, Intrinsic::ctpop, NotOp);
908
909 Value *R = nullptr;
910
911 // Do the transformation here to avoid potentially introducing an infinite
912 // loop.
913 switch (Opc) {
914 case Instruction::Sub:
915 R = Builder.CreateAdd(CtpopOfNotOp, ConstantExpr::getSub(C, BitWidthC));
916 break;
917 case Instruction::Or:
918 case Instruction::Add:
919 R = Builder.CreateSub(ConstantExpr::getAdd(C, BitWidthC), CtpopOfNotOp);
920 break;
921 case Instruction::ICmp:
922 R = Builder.CreateICmp(cast<ICmpInst>(I)->getSwappedPredicate(),
923 CtpopOfNotOp, ConstantExpr::getSub(BitWidthC, C));
924 break;
925 default:
926 llvm_unreachable("Unhandled Opcode");
927 }
928 assert(R != nullptr);
929 return replaceInstUsesWith(*I, R);
930}
931
932// (Binop1 (Binop2 (logic_shift X, C), C1), (logic_shift Y, C))
933// IFF
934// 1) the logic_shifts match
935// 2) either both binops are binops and one is `and` or
936// BinOp1 is `and`
937// (logic_shift (inv_logic_shift C1, C), C) == C1 or
938//
939// -> (logic_shift (Binop1 (Binop2 X, inv_logic_shift(C1, C)), Y), C)
940//
941// (Binop1 (Binop2 (logic_shift X, Amt), Mask), (logic_shift Y, Amt))
942// IFF
943// 1) the logic_shifts match
944// 2) BinOp1 == BinOp2 (if BinOp == `add`, then also requires `shl`).
945//
946// -> (BinOp (logic_shift (BinOp X, Y)), Mask)
947//
948// (Binop1 (Binop2 (arithmetic_shift X, Amt), Mask), (arithmetic_shift Y, Amt))
949// IFF
950// 1) Binop1 is bitwise logical operator `and`, `or` or `xor`
951// 2) Binop2 is `not`
952//
953// -> (arithmetic_shift Binop1((not X), Y), Amt)
954
956 const DataLayout &DL = I.getDataLayout();
957 auto IsValidBinOpc = [](unsigned Opc) {
958 switch (Opc) {
959 default:
960 return false;
961 case Instruction::And:
962 case Instruction::Or:
963 case Instruction::Xor:
964 case Instruction::Add:
965 // Skip Sub as we only match constant masks which will canonicalize to use
966 // add.
967 return true;
968 }
969 };
970
971 // Check if we can distribute binop arbitrarily. `add` + `lshr` has extra
972 // constraints.
973 auto IsCompletelyDistributable = [](unsigned BinOpc1, unsigned BinOpc2,
974 unsigned ShOpc) {
975 assert(ShOpc != Instruction::AShr);
976 return (BinOpc1 != Instruction::Add && BinOpc2 != Instruction::Add) ||
977 ShOpc == Instruction::Shl;
978 };
979
980 auto GetInvShift = [](unsigned ShOpc) {
981 assert(ShOpc != Instruction::AShr);
982 return ShOpc == Instruction::LShr ? Instruction::Shl : Instruction::LShr;
983 };
984
985 auto CanDistributeBinops = [&](unsigned BinOpc1, unsigned BinOpc2,
986 unsigned ShOpc, Constant *CMask,
987 Constant *CShift) {
988 // If the BinOp1 is `and` we don't need to check the mask.
989 if (BinOpc1 == Instruction::And)
990 return true;
991
992 // For all other possible transfers we need complete distributable
993 // binop/shift (anything but `add` + `lshr`).
994 if (!IsCompletelyDistributable(BinOpc1, BinOpc2, ShOpc))
995 return false;
996
997 // If BinOp2 is `and`, any mask works (this only really helps for non-splat
998 // vecs, otherwise the mask will be simplified and the following check will
999 // handle it).
1000 if (BinOpc2 == Instruction::And)
1001 return true;
1002
1003 // Otherwise, need mask that meets the below requirement.
1004 // (logic_shift (inv_logic_shift Mask, ShAmt), ShAmt) == Mask
1005 Constant *MaskInvShift =
1006 ConstantFoldBinaryOpOperands(GetInvShift(ShOpc), CMask, CShift, DL);
1007 return ConstantFoldBinaryOpOperands(ShOpc, MaskInvShift, CShift, DL) ==
1008 CMask;
1009 };
1010
1011 auto MatchBinOp = [&](unsigned ShOpnum) -> Instruction * {
1012 Constant *CMask, *CShift;
1013 Value *X, *Y, *ShiftedX, *Mask, *Shift;
1014 if (!match(I.getOperand(ShOpnum),
1015 m_OneUse(m_Shift(m_Value(Y), m_Value(Shift)))))
1016 return nullptr;
1017 if (!match(I.getOperand(1 - ShOpnum),
1019 m_OneUse(m_Shift(m_Value(X), m_Specific(Shift))),
1020 m_Value(ShiftedX)),
1021 m_Value(Mask))))
1022 return nullptr;
1023 // Make sure we are matching instruction shifts and not ConstantExpr
1024 auto *IY = dyn_cast<Instruction>(I.getOperand(ShOpnum));
1025 auto *IX = dyn_cast<Instruction>(ShiftedX);
1026 if (!IY || !IX)
1027 return nullptr;
1028
1029 // LHS and RHS need same shift opcode
1030 unsigned ShOpc = IY->getOpcode();
1031 if (ShOpc != IX->getOpcode())
1032 return nullptr;
1033
1034 // Make sure binop is real instruction and not ConstantExpr
1035 auto *BO2 = dyn_cast<Instruction>(I.getOperand(1 - ShOpnum));
1036 if (!BO2)
1037 return nullptr;
1038
1039 unsigned BinOpc = BO2->getOpcode();
1040 // Make sure we have valid binops.
1041 if (!IsValidBinOpc(I.getOpcode()) || !IsValidBinOpc(BinOpc))
1042 return nullptr;
1043
1044 if (ShOpc == Instruction::AShr) {
1045 if (Instruction::isBitwiseLogicOp(I.getOpcode()) &&
1046 BinOpc == Instruction::Xor && match(Mask, m_AllOnes())) {
1047 Value *NotX = Builder.CreateNot(X);
1048 Value *NewBinOp = Builder.CreateBinOp(I.getOpcode(), Y, NotX);
1050 static_cast<Instruction::BinaryOps>(ShOpc), NewBinOp, Shift);
1051 }
1052
1053 return nullptr;
1054 }
1055
1056 // If BinOp1 == BinOp2 and it's bitwise or shl with add, then just
1057 // distribute to drop the shift irrelevant of constants.
1058 if (BinOpc == I.getOpcode() &&
1059 IsCompletelyDistributable(I.getOpcode(), BinOpc, ShOpc)) {
1060 Value *NewBinOp2 = Builder.CreateBinOp(I.getOpcode(), X, Y);
1061 Value *NewBinOp1 = Builder.CreateBinOp(
1062 static_cast<Instruction::BinaryOps>(ShOpc), NewBinOp2, Shift);
1063 return BinaryOperator::Create(I.getOpcode(), NewBinOp1, Mask);
1064 }
1065
1066 // Otherwise we can only distribute by constant shifting the mask, so
1067 // ensure we have constants.
1068 if (!match(Shift, m_ImmConstant(CShift)))
1069 return nullptr;
1070 if (!match(Mask, m_ImmConstant(CMask)))
1071 return nullptr;
1072
1073 // Check if we can distribute the binops.
1074 if (!CanDistributeBinops(I.getOpcode(), BinOpc, ShOpc, CMask, CShift))
1075 return nullptr;
1076
1077 Constant *NewCMask =
1078 ConstantFoldBinaryOpOperands(GetInvShift(ShOpc), CMask, CShift, DL);
1079 Value *NewBinOp2 = Builder.CreateBinOp(
1080 static_cast<Instruction::BinaryOps>(BinOpc), X, NewCMask);
1081 Value *NewBinOp1 = Builder.CreateBinOp(I.getOpcode(), Y, NewBinOp2);
1082 return BinaryOperator::Create(static_cast<Instruction::BinaryOps>(ShOpc),
1083 NewBinOp1, CShift);
1084 };
1085
1086 if (Instruction *R = MatchBinOp(0))
1087 return R;
1088 return MatchBinOp(1);
1089}
1090
1091// (Binop (zext C), (select C, T, F))
1092// -> (select C, (binop 1, T), (binop 0, F))
1093//
1094// (Binop (sext C), (select C, T, F))
1095// -> (select C, (binop -1, T), (binop 0, F))
1096//
1097// Attempt to simplify binary operations into a select with folded args, when
1098// one operand of the binop is a select instruction and the other operand is a
1099// zext/sext extension, whose value is the select condition.
1102 // TODO: this simplification may be extended to any speculatable instruction,
1103 // not just binops, and would possibly be handled better in FoldOpIntoSelect.
1104 Instruction::BinaryOps Opc = I.getOpcode();
1105 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
1106 Value *A, *CondVal, *TrueVal, *FalseVal;
1107 Value *CastOp;
1108
1109 auto MatchSelectAndCast = [&](Value *CastOp, Value *SelectOp) {
1110 return match(CastOp, m_ZExtOrSExt(m_Value(A))) &&
1111 A->getType()->getScalarSizeInBits() == 1 &&
1112 match(SelectOp, m_Select(m_Value(CondVal), m_Value(TrueVal),
1113 m_Value(FalseVal)));
1114 };
1115
1116 // Make sure one side of the binop is a select instruction, and the other is a
1117 // zero/sign extension operating on a i1.
1118 if (MatchSelectAndCast(LHS, RHS))
1119 CastOp = LHS;
1120 else if (MatchSelectAndCast(RHS, LHS))
1121 CastOp = RHS;
1122 else
1123 return nullptr;
1124
1125 auto NewFoldedConst = [&](bool IsTrueArm, Value *V) {
1126 bool IsCastOpRHS = (CastOp == RHS);
1127 bool IsZExt = isa<ZExtInst>(CastOp);
1128 Constant *C;
1129
1130 if (IsTrueArm) {
1131 C = Constant::getNullValue(V->getType());
1132 } else if (IsZExt) {
1133 unsigned BitWidth = V->getType()->getScalarSizeInBits();
1134 C = Constant::getIntegerValue(V->getType(), APInt(BitWidth, 1));
1135 } else {
1136 C = Constant::getAllOnesValue(V->getType());
1137 }
1138
1139 return IsCastOpRHS ? Builder.CreateBinOp(Opc, V, C)
1140 : Builder.CreateBinOp(Opc, C, V);
1141 };
1142
1143 // If the value used in the zext/sext is the select condition, or the negated
1144 // of the select condition, the binop can be simplified.
1145 if (CondVal == A) {
1146 Value *NewTrueVal = NewFoldedConst(false, TrueVal);
1147 return SelectInst::Create(CondVal, NewTrueVal,
1148 NewFoldedConst(true, FalseVal));
1149 }
1150
1151 if (match(A, m_Not(m_Specific(CondVal)))) {
1152 Value *NewTrueVal = NewFoldedConst(true, TrueVal);
1153 return SelectInst::Create(CondVal, NewTrueVal,
1154 NewFoldedConst(false, FalseVal));
1155 }
1156
1157 return nullptr;
1158}
1159
1161 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
1164 Instruction::BinaryOps TopLevelOpcode = I.getOpcode();
1165 Value *A, *B, *C, *D;
1166 Instruction::BinaryOps LHSOpcode, RHSOpcode;
1167
1168 if (Op0)
1169 LHSOpcode = getBinOpsForFactorization(TopLevelOpcode, Op0, A, B, Op1);
1170 if (Op1)
1171 RHSOpcode = getBinOpsForFactorization(TopLevelOpcode, Op1, C, D, Op0);
1172
1173 // The instruction has the form "(A op' B) op (C op' D)". Try to factorize
1174 // a common term.
1175 if (Op0 && Op1 && LHSOpcode == RHSOpcode)
1176 if (Value *V = tryFactorization(I, SQ, Builder, LHSOpcode, A, B, C, D))
1177 return V;
1178
1179 // The instruction has the form "(A op' B) op (C)". Try to factorize common
1180 // term.
1181 if (Op0)
1182 if (Value *Ident = getIdentityValue(LHSOpcode, RHS))
1183 if (Value *V =
1184 tryFactorization(I, SQ, Builder, LHSOpcode, A, B, RHS, Ident))
1185 return V;
1186
1187 // The instruction has the form "(B) op (C op' D)". Try to factorize common
1188 // term.
1189 if (Op1)
1190 if (Value *Ident = getIdentityValue(RHSOpcode, LHS))
1191 if (Value *V =
1192 tryFactorization(I, SQ, Builder, RHSOpcode, LHS, Ident, C, D))
1193 return V;
1194
1195 return nullptr;
1196}
1197
1198/// This tries to simplify binary operations which some other binary operation
1199/// distributes over either by factorizing out common terms
1200/// (eg "(A*B)+(A*C)" -> "A*(B+C)") or expanding out if this results in
1201/// simplifications (eg: "A & (B | C) -> (A&B) | (A&C)" if this is a win).
1202/// Returns the simplified value, or null if it didn't simplify.
1204 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
1207 Instruction::BinaryOps TopLevelOpcode = I.getOpcode();
1208
1209 // Factorization.
1210 if (Value *R = tryFactorizationFolds(I))
1211 return R;
1212
1213 // Expansion.
1214 if (Op0 && rightDistributesOverLeft(Op0->getOpcode(), TopLevelOpcode)) {
1215 // The instruction has the form "(A op' B) op C". See if expanding it out
1216 // to "(A op C) op' (B op C)" results in simplifications.
1217 Value *A = Op0->getOperand(0), *B = Op0->getOperand(1), *C = RHS;
1218 Instruction::BinaryOps InnerOpcode = Op0->getOpcode(); // op'
1219
1220 // Disable the use of undef because it's not safe to distribute undef.
1221 auto SQDistributive = SQ.getWithInstruction(&I).getWithoutUndef();
1222 Value *L = simplifyBinOp(TopLevelOpcode, A, C, SQDistributive);
1223 Value *R = simplifyBinOp(TopLevelOpcode, B, C, SQDistributive);
1224
1225 // Do "A op C" and "B op C" both simplify?
1226 if (L && R) {
1227 // They do! Return "L op' R".
1228 ++NumExpand;
1229 C = Builder.CreateBinOp(InnerOpcode, L, R);
1230 C->takeName(&I);
1231 return C;
1232 }
1233
1234 // Does "A op C" simplify to the identity value for the inner opcode?
1235 if (L && L == ConstantExpr::getBinOpIdentity(InnerOpcode, L->getType())) {
1236 // They do! Return "B op C".
1237 ++NumExpand;
1238 C = Builder.CreateBinOp(TopLevelOpcode, B, C);
1239 C->takeName(&I);
1240 return C;
1241 }
1242
1243 // Does "B op C" simplify to the identity value for the inner opcode?
1244 if (R && R == ConstantExpr::getBinOpIdentity(InnerOpcode, R->getType())) {
1245 // They do! Return "A op C".
1246 ++NumExpand;
1247 C = Builder.CreateBinOp(TopLevelOpcode, A, C);
1248 C->takeName(&I);
1249 return C;
1250 }
1251 }
1252
1253 if (Op1 && leftDistributesOverRight(TopLevelOpcode, Op1->getOpcode())) {
1254 // The instruction has the form "A op (B op' C)". See if expanding it out
1255 // to "(A op B) op' (A op C)" results in simplifications.
1256 Value *A = LHS, *B = Op1->getOperand(0), *C = Op1->getOperand(1);
1257 Instruction::BinaryOps InnerOpcode = Op1->getOpcode(); // op'
1258
1259 // Disable the use of undef because it's not safe to distribute undef.
1260 auto SQDistributive = SQ.getWithInstruction(&I).getWithoutUndef();
1261 Value *L = simplifyBinOp(TopLevelOpcode, A, B, SQDistributive);
1262 Value *R = simplifyBinOp(TopLevelOpcode, A, C, SQDistributive);
1263
1264 // Do "A op B" and "A op C" both simplify?
1265 if (L && R) {
1266 // They do! Return "L op' R".
1267 ++NumExpand;
1268 A = Builder.CreateBinOp(InnerOpcode, L, R);
1269 A->takeName(&I);
1270 return A;
1271 }
1272
1273 // Does "A op B" simplify to the identity value for the inner opcode?
1274 if (L && L == ConstantExpr::getBinOpIdentity(InnerOpcode, L->getType())) {
1275 // They do! Return "A op C".
1276 ++NumExpand;
1277 A = Builder.CreateBinOp(TopLevelOpcode, A, C);
1278 A->takeName(&I);
1279 return A;
1280 }
1281
1282 // Does "A op C" simplify to the identity value for the inner opcode?
1283 if (R && R == ConstantExpr::getBinOpIdentity(InnerOpcode, R->getType())) {
1284 // They do! Return "A op B".
1285 ++NumExpand;
1286 A = Builder.CreateBinOp(TopLevelOpcode, A, B);
1287 A->takeName(&I);
1288 return A;
1289 }
1290 }
1291
1292 return SimplifySelectsFeedingBinaryOp(I, LHS, RHS);
1293}
1294
1295static std::optional<std::pair<Value *, Value *>>
1297 if (LHS->getParent() != RHS->getParent())
1298 return std::nullopt;
1299
1300 if (LHS->getNumIncomingValues() < 2)
1301 return std::nullopt;
1302
1303 if (!equal(LHS->blocks(), RHS->blocks()))
1304 return std::nullopt;
1305
1306 Value *L0 = LHS->getIncomingValue(0);
1307 Value *R0 = RHS->getIncomingValue(0);
1308
1309 for (unsigned I = 1, E = LHS->getNumIncomingValues(); I != E; ++I) {
1310 Value *L1 = LHS->getIncomingValue(I);
1311 Value *R1 = RHS->getIncomingValue(I);
1312
1313 if ((L0 == L1 && R0 == R1) || (L0 == R1 && R0 == L1))
1314 continue;
1315
1316 return std::nullopt;
1317 }
1318
1319 return std::optional(std::pair(L0, R0));
1320}
1321
1322std::optional<std::pair<Value *, Value *>>
1323InstCombinerImpl::matchSymmetricPair(Value *LHS, Value *RHS) {
1326 if (!LHSInst || !RHSInst || LHSInst->getOpcode() != RHSInst->getOpcode())
1327 return std::nullopt;
1328 switch (LHSInst->getOpcode()) {
1329 case Instruction::PHI:
1331 case Instruction::Select: {
1332 Value *Cond = LHSInst->getOperand(0);
1333 Value *TrueVal = LHSInst->getOperand(1);
1334 Value *FalseVal = LHSInst->getOperand(2);
1335 if (Cond == RHSInst->getOperand(0) && TrueVal == RHSInst->getOperand(2) &&
1336 FalseVal == RHSInst->getOperand(1))
1337 return std::pair(TrueVal, FalseVal);
1338 return std::nullopt;
1339 }
1340 case Instruction::Call: {
1341 // Match min(a, b) and max(a, b)
1342 MinMaxIntrinsic *LHSMinMax = dyn_cast<MinMaxIntrinsic>(LHSInst);
1343 MinMaxIntrinsic *RHSMinMax = dyn_cast<MinMaxIntrinsic>(RHSInst);
1344 if (LHSMinMax && RHSMinMax &&
1345 LHSMinMax->getPredicate() ==
1347 ((LHSMinMax->getLHS() == RHSMinMax->getLHS() &&
1348 LHSMinMax->getRHS() == RHSMinMax->getRHS()) ||
1349 (LHSMinMax->getLHS() == RHSMinMax->getRHS() &&
1350 LHSMinMax->getRHS() == RHSMinMax->getLHS())))
1351 return std::pair(LHSMinMax->getLHS(), LHSMinMax->getRHS());
1352 return std::nullopt;
1353 }
1354 default:
1355 return std::nullopt;
1356 }
1357}
1358
1360 Value *LHS,
1361 Value *RHS) {
1362 Value *A, *B, *C, *D, *E, *F;
1363 bool LHSIsSelect = match(LHS, m_Select(m_Value(A), m_Value(B), m_Value(C)));
1364 bool RHSIsSelect = match(RHS, m_Select(m_Value(D), m_Value(E), m_Value(F)));
1365 if (!LHSIsSelect && !RHSIsSelect)
1366 return nullptr;
1367
1369 ? nullptr
1370 : cast<SelectInst>(LHSIsSelect ? LHS : RHS);
1371
1372 FastMathFlags FMF;
1374 if (isa<FPMathOperator>(&I)) {
1375 FMF = I.getFastMathFlags();
1376 Builder.setFastMathFlags(FMF);
1377 }
1378
1379 Instruction::BinaryOps Opcode = I.getOpcode();
1380 SimplifyQuery Q = SQ.getWithInstruction(&I);
1381
1382 Value *Cond, *True = nullptr, *False = nullptr;
1383
1384 // Special-case for add/negate combination. Replace the zero in the negation
1385 // with the trailing add operand:
1386 // (Cond ? TVal : -N) + Z --> Cond ? True : (Z - N)
1387 // (Cond ? -N : FVal) + Z --> Cond ? (Z - N) : False
1388 auto foldAddNegate = [&](Value *TVal, Value *FVal, Value *Z) -> Value * {
1389 // We need an 'add' and exactly 1 arm of the select to have been simplified.
1390 if (Opcode != Instruction::Add || (!True && !False) || (True && False))
1391 return nullptr;
1392 Value *N;
1393 if (True && match(FVal, m_Neg(m_Value(N)))) {
1394 Value *Sub = Builder.CreateSub(Z, N);
1395 return Builder.CreateSelect(Cond, True, Sub, I.getName(), SI);
1396 }
1397 if (False && match(TVal, m_Neg(m_Value(N)))) {
1398 Value *Sub = Builder.CreateSub(Z, N);
1399 return Builder.CreateSelect(Cond, Sub, False, I.getName(), SI);
1400 }
1401 return nullptr;
1402 };
1403
1404 if (LHSIsSelect && RHSIsSelect && A == D) {
1405 // (A ? B : C) op (A ? E : F) -> A ? (B op E) : (C op F)
1406 Cond = A;
1407 True = simplifyBinOp(Opcode, B, E, FMF, Q);
1408 False = simplifyBinOp(Opcode, C, F, FMF, Q);
1409
1410 if (LHS->hasOneUse() && RHS->hasOneUse()) {
1411 if (False && !True)
1412 True = Builder.CreateBinOp(Opcode, B, E);
1413 else if (True && !False)
1414 False = Builder.CreateBinOp(Opcode, C, F);
1415 }
1416 } else if (LHSIsSelect && LHS->hasOneUse()) {
1417 // (A ? B : C) op Y -> A ? (B op Y) : (C op Y)
1418 Cond = A;
1419 True = simplifyBinOp(Opcode, B, RHS, FMF, Q);
1420 False = simplifyBinOp(Opcode, C, RHS, FMF, Q);
1421 if (Value *NewSel = foldAddNegate(B, C, RHS))
1422 return NewSel;
1423 } else if (RHSIsSelect && RHS->hasOneUse()) {
1424 // X op (D ? E : F) -> D ? (X op E) : (X op F)
1425 Cond = D;
1426 True = simplifyBinOp(Opcode, LHS, E, FMF, Q);
1427 False = simplifyBinOp(Opcode, LHS, F, FMF, Q);
1428 if (Value *NewSel = foldAddNegate(E, F, LHS))
1429 return NewSel;
1430 }
1431
1432 if (!True || !False)
1433 return nullptr;
1434
1435 Value *NewSI = Builder.CreateSelect(Cond, True, False, I.getName(), SI);
1436 NewSI->takeName(&I);
1437 return NewSI;
1438}
1439
1440/// Freely adapt every user of V as-if V was changed to !V.
1441/// WARNING: only if canFreelyInvertAllUsersOf() said this can be done.
1443 assert(!isa<Constant>(I) && "Shouldn't invert users of constant");
1444 for (User *U : make_early_inc_range(I->users())) {
1445 if (U == IgnoredUser)
1446 continue; // Don't consider this user.
1447 switch (cast<Instruction>(U)->getOpcode()) {
1448 case Instruction::Select: {
1449 auto *SI = cast<SelectInst>(U);
1450 SI->swapValues();
1451 SI->swapProfMetadata();
1452 break;
1453 }
1454 case Instruction::Br: {
1456 BI->swapSuccessors(); // swaps prof metadata too
1457 if (BPI)
1458 BPI->swapSuccEdgesProbabilities(BI->getParent());
1459 break;
1460 }
1461 case Instruction::Xor:
1463 // Add to worklist for DCE.
1465 break;
1466 default:
1467 llvm_unreachable("Got unexpected user - out of sync with "
1468 "canFreelyInvertAllUsersOf() ?");
1469 }
1470 }
1471
1472 // Update pre-existing debug value uses.
1473 SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
1474 llvm::findDbgValues(I, DbgVariableRecords);
1475
1476 for (DbgVariableRecord *DbgVal : DbgVariableRecords) {
1477 SmallVector<uint64_t, 1> Ops = {dwarf::DW_OP_not};
1478 for (unsigned Idx = 0, End = DbgVal->getNumVariableLocationOps();
1479 Idx != End; ++Idx)
1480 if (DbgVal->getVariableLocationOp(Idx) == I)
1481 DbgVal->setExpression(
1482 DIExpression::appendOpsToArg(DbgVal->getExpression(), Ops, Idx));
1483 }
1484}
1485
1486/// Given a 'sub' instruction, return the RHS of the instruction if the LHS is a
1487/// constant zero (which is the 'negate' form).
1488Value *InstCombinerImpl::dyn_castNegVal(Value *V) const {
1489 Value *NegV;
1490 if (match(V, m_Neg(m_Value(NegV))))
1491 return NegV;
1492
1493 // Constants can be considered to be negated values if they can be folded.
1495 return ConstantExpr::getNeg(C);
1496
1498 if (C->getType()->getElementType()->isIntegerTy())
1499 return ConstantExpr::getNeg(C);
1500
1502 for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
1503 Constant *Elt = CV->getAggregateElement(i);
1504 if (!Elt)
1505 return nullptr;
1506
1507 if (isa<UndefValue>(Elt))
1508 continue;
1509
1510 if (!isa<ConstantInt>(Elt))
1511 return nullptr;
1512 }
1513 return ConstantExpr::getNeg(CV);
1514 }
1515
1516 // Negate integer vector splats.
1517 if (auto *CV = dyn_cast<Constant>(V))
1518 if (CV->getType()->isVectorTy() &&
1519 CV->getType()->getScalarType()->isIntegerTy() && CV->getSplatValue())
1520 return ConstantExpr::getNeg(CV);
1521
1522 return nullptr;
1523}
1524
1525// Try to fold:
1526// 1) (fp_binop ({s|u}itofp x), ({s|u}itofp y))
1527// -> ({s|u}itofp (int_binop x, y))
1528// 2) (fp_binop ({s|u}itofp x), FpC)
1529// -> ({s|u}itofp (int_binop x, (fpto{s|u}i FpC)))
1530//
1531// Assuming the sign of the cast for x/y is `OpsFromSigned`.
1532Instruction *InstCombinerImpl::foldFBinOpOfIntCastsFromSign(
1533 BinaryOperator &BO, bool OpsFromSigned, std::array<Value *, 2> IntOps,
1535
1536 Type *FPTy = BO.getType();
1537 Type *IntTy = IntOps[0]->getType();
1538
1539 unsigned IntSz = IntTy->getScalarSizeInBits();
1540 // This is the maximum number of inuse bits by the integer where the int -> fp
1541 // casts are exact.
1542 unsigned MaxRepresentableBits =
1544
1545 // Preserve known number of leading bits. This can allow us to trivial nsw/nuw
1546 // checks later on.
1547 unsigned NumUsedLeadingBits[2] = {IntSz, IntSz};
1548
1549 // NB: This only comes up if OpsFromSigned is true, so there is no need to
1550 // cache if between calls to `foldFBinOpOfIntCastsFromSign`.
1551 auto IsNonZero = [&](unsigned OpNo) -> bool {
1552 if (OpsKnown[OpNo].hasKnownBits() &&
1553 OpsKnown[OpNo].getKnownBits(SQ).isNonZero())
1554 return true;
1555 return isKnownNonZero(IntOps[OpNo], SQ);
1556 };
1557
1558 auto IsNonNeg = [&](unsigned OpNo) -> bool {
1559 // NB: This matches the impl in ValueTracking, we just try to use cached
1560 // knownbits here. If we ever start supporting WithCache for
1561 // `isKnownNonNegative`, change this to an explicit call.
1562 return OpsKnown[OpNo].getKnownBits(SQ).isNonNegative();
1563 };
1564
1565 // Check if we know for certain that ({s|u}itofp op) is exact.
1566 auto IsValidPromotion = [&](unsigned OpNo) -> bool {
1567 // Can we treat this operand as the desired sign?
1568 if (OpsFromSigned != isa<SIToFPInst>(BO.getOperand(OpNo)) &&
1569 !IsNonNeg(OpNo))
1570 return false;
1571
1572 // If fp precision >= bitwidth(op) then its exact.
1573 // NB: This is slightly conservative for `sitofp`. For signed conversion, we
1574 // can handle `MaxRepresentableBits == IntSz - 1` as the sign bit will be
1575 // handled specially. We can't, however, increase the bound arbitrarily for
1576 // `sitofp` as for larger sizes, it won't sign extend.
1577 if (MaxRepresentableBits < IntSz) {
1578 // Otherwise if its signed cast check that fp precisions >= bitwidth(op) -
1579 // numSignBits(op).
1580 // TODO: If we add support for `WithCache` in `ComputeNumSignBits`, change
1581 // `IntOps[OpNo]` arguments to `KnownOps[OpNo]`.
1582 if (OpsFromSigned)
1583 NumUsedLeadingBits[OpNo] = IntSz - ComputeNumSignBits(IntOps[OpNo]);
1584 // Finally for unsigned check that fp precision >= bitwidth(op) -
1585 // numLeadingZeros(op).
1586 else {
1587 NumUsedLeadingBits[OpNo] =
1588 IntSz - OpsKnown[OpNo].getKnownBits(SQ).countMinLeadingZeros();
1589 }
1590 }
1591 // NB: We could also check if op is known to be a power of 2 or zero (which
1592 // will always be representable). Its unlikely, however, that is we are
1593 // unable to bound op in any way we will be able to pass the overflow checks
1594 // later on.
1595
1596 if (MaxRepresentableBits < NumUsedLeadingBits[OpNo])
1597 return false;
1598 // Signed + Mul also requires that op is non-zero to avoid -0 cases.
1599 return !OpsFromSigned || BO.getOpcode() != Instruction::FMul ||
1600 IsNonZero(OpNo);
1601 };
1602
1603 // If we have a constant rhs, see if we can losslessly convert it to an int.
1604 if (Op1FpC != nullptr) {
1605 // Signed + Mul req non-zero
1606 if (OpsFromSigned && BO.getOpcode() == Instruction::FMul &&
1607 !match(Op1FpC, m_NonZeroFP()))
1608 return nullptr;
1609
1611 OpsFromSigned ? Instruction::FPToSI : Instruction::FPToUI, Op1FpC,
1612 IntTy, DL);
1613 if (Op1IntC == nullptr)
1614 return nullptr;
1615 if (ConstantFoldCastOperand(OpsFromSigned ? Instruction::SIToFP
1616 : Instruction::UIToFP,
1617 Op1IntC, FPTy, DL) != Op1FpC)
1618 return nullptr;
1619
1620 // First try to keep sign of cast the same.
1621 IntOps[1] = Op1IntC;
1622 }
1623
1624 // Ensure lhs/rhs integer types match.
1625 if (IntTy != IntOps[1]->getType())
1626 return nullptr;
1627
1628 if (Op1FpC == nullptr) {
1629 if (!IsValidPromotion(1))
1630 return nullptr;
1631 }
1632 if (!IsValidPromotion(0))
1633 return nullptr;
1634
1635 // Final we check if the integer version of the binop will not overflow.
1637 // Because of the precision check, we can often rule out overflows.
1638 bool NeedsOverflowCheck = true;
1639 // Try to conservatively rule out overflow based on the already done precision
1640 // checks.
1641 unsigned OverflowMaxOutputBits = OpsFromSigned ? 2 : 1;
1642 unsigned OverflowMaxCurBits =
1643 std::max(NumUsedLeadingBits[0], NumUsedLeadingBits[1]);
1644 bool OutputSigned = OpsFromSigned;
1645 switch (BO.getOpcode()) {
1646 case Instruction::FAdd:
1647 IntOpc = Instruction::Add;
1648 OverflowMaxOutputBits += OverflowMaxCurBits;
1649 break;
1650 case Instruction::FSub:
1651 IntOpc = Instruction::Sub;
1652 OverflowMaxOutputBits += OverflowMaxCurBits;
1653 break;
1654 case Instruction::FMul:
1655 IntOpc = Instruction::Mul;
1656 OverflowMaxOutputBits += OverflowMaxCurBits * 2;
1657 break;
1658 default:
1659 llvm_unreachable("Unsupported binop");
1660 }
1661 // The precision check may have already ruled out overflow.
1662 if (OverflowMaxOutputBits < IntSz) {
1663 NeedsOverflowCheck = false;
1664 // We can bound unsigned overflow from sub to in range signed value (this is
1665 // what allows us to avoid the overflow check for sub).
1666 if (IntOpc == Instruction::Sub)
1667 OutputSigned = true;
1668 }
1669
1670 // Precision check did not rule out overflow, so need to check.
1671 // TODO: If we add support for `WithCache` in `willNotOverflow`, change
1672 // `IntOps[...]` arguments to `KnownOps[...]`.
1673 if (NeedsOverflowCheck &&
1674 !willNotOverflow(IntOpc, IntOps[0], IntOps[1], BO, OutputSigned))
1675 return nullptr;
1676
1677 Value *IntBinOp = Builder.CreateBinOp(IntOpc, IntOps[0], IntOps[1]);
1678 if (auto *IntBO = dyn_cast<BinaryOperator>(IntBinOp)) {
1679 IntBO->setHasNoSignedWrap(OutputSigned);
1680 IntBO->setHasNoUnsignedWrap(!OutputSigned);
1681 }
1682 if (OutputSigned)
1683 return new SIToFPInst(IntBinOp, FPTy);
1684 return new UIToFPInst(IntBinOp, FPTy);
1685}
1686
1687// Try to fold:
1688// 1) (fp_binop ({s|u}itofp x), ({s|u}itofp y))
1689// -> ({s|u}itofp (int_binop x, y))
1690// 2) (fp_binop ({s|u}itofp x), FpC)
1691// -> ({s|u}itofp (int_binop x, (fpto{s|u}i FpC)))
1692Instruction *InstCombinerImpl::foldFBinOpOfIntCasts(BinaryOperator &BO) {
1693 // Don't perform the fold on vectors, as the integer operation may be much
1694 // more expensive than the float operation in that case.
1695 if (BO.getType()->isVectorTy())
1696 return nullptr;
1697
1698 std::array<Value *, 2> IntOps = {nullptr, nullptr};
1699 Constant *Op1FpC = nullptr;
1700 // Check for:
1701 // 1) (binop ({s|u}itofp x), ({s|u}itofp y))
1702 // 2) (binop ({s|u}itofp x), FpC)
1703 if (!match(BO.getOperand(0), m_SIToFP(m_Value(IntOps[0]))) &&
1704 !match(BO.getOperand(0), m_UIToFP(m_Value(IntOps[0]))))
1705 return nullptr;
1706
1707 if (!match(BO.getOperand(1), m_Constant(Op1FpC)) &&
1708 !match(BO.getOperand(1), m_SIToFP(m_Value(IntOps[1]))) &&
1709 !match(BO.getOperand(1), m_UIToFP(m_Value(IntOps[1]))))
1710 return nullptr;
1711
1712 // Cache KnownBits a bit to potentially save some analysis.
1713 SmallVector<WithCache<const Value *>, 2> OpsKnown = {IntOps[0], IntOps[1]};
1714
1715 // Try treating x/y as coming from both `uitofp` and `sitofp`. There are
1716 // different constraints depending on the sign of the cast.
1717 // NB: `(uitofp nneg X)` == `(sitofp nneg X)`.
1718 if (Instruction *R = foldFBinOpOfIntCastsFromSign(BO, /*OpsFromSigned=*/false,
1719 IntOps, Op1FpC, OpsKnown))
1720 return R;
1721 return foldFBinOpOfIntCastsFromSign(BO, /*OpsFromSigned=*/true, IntOps,
1722 Op1FpC, OpsKnown);
1723}
1724
1725/// A binop with a constant operand and a sign-extended boolean operand may be
1726/// converted into a select of constants by applying the binary operation to
1727/// the constant with the two possible values of the extended boolean (0 or -1).
1728Instruction *InstCombinerImpl::foldBinopOfSextBoolToSelect(BinaryOperator &BO) {
1729 // TODO: Handle non-commutative binop (constant is operand 0).
1730 // TODO: Handle zext.
1731 // TODO: Peek through 'not' of cast.
1732 Value *BO0 = BO.getOperand(0);
1733 Value *BO1 = BO.getOperand(1);
1734 Value *X;
1735 Constant *C;
1736 if (!match(BO0, m_SExt(m_Value(X))) || !match(BO1, m_ImmConstant(C)) ||
1737 !X->getType()->isIntOrIntVectorTy(1))
1738 return nullptr;
1739
1740 // bo (sext i1 X), C --> select X, (bo -1, C), (bo 0, C)
1743 Value *TVal = Builder.CreateBinOp(BO.getOpcode(), Ones, C);
1744 Value *FVal = Builder.CreateBinOp(BO.getOpcode(), Zero, C);
1745 return createSelectInstWithUnknownProfile(X, TVal, FVal);
1746}
1747
1749 bool IsTrueArm) {
1751 for (Value *Op : I.operands()) {
1752 Value *V = nullptr;
1753 if (Op == SI) {
1754 V = IsTrueArm ? SI->getTrueValue() : SI->getFalseValue();
1755 } else if (match(SI->getCondition(),
1758 m_Specific(Op), m_Value(V))) &&
1760 // Pass
1761 } else {
1762 V = Op;
1763 }
1764 Ops.push_back(V);
1765 }
1766
1767 return simplifyInstructionWithOperands(&I, Ops, I.getDataLayout());
1768}
1769
1771 Value *NewOp, InstCombiner &IC) {
1772 Instruction *Clone = I.clone();
1773 Clone->replaceUsesOfWith(SI, NewOp);
1775 IC.InsertNewInstBefore(Clone, I.getIterator());
1776 return Clone;
1777}
1778
1780 bool FoldWithMultiUse,
1781 bool SimplifyBothArms) {
1782 // Don't modify shared select instructions unless set FoldWithMultiUse
1783 if (!SI->hasOneUse() && !FoldWithMultiUse)
1784 return nullptr;
1785
1786 Value *TV = SI->getTrueValue();
1787 Value *FV = SI->getFalseValue();
1788
1789 // Bool selects with constant operands can be folded to logical ops.
1790 if (SI->getType()->isIntOrIntVectorTy(1))
1791 return nullptr;
1792
1793 // Avoid breaking min/max reduction pattern,
1794 // which is necessary for vectorization later.
1796 for (Value *IntrinOp : Op.operands())
1797 if (auto *PN = dyn_cast<PHINode>(IntrinOp))
1798 for (Value *PhiOp : PN->operands())
1799 if (PhiOp == &Op)
1800 return nullptr;
1801
1802 // Test if a FCmpInst instruction is used exclusively by a select as
1803 // part of a minimum or maximum operation. If so, refrain from doing
1804 // any other folding. This helps out other analyses which understand
1805 // non-obfuscated minimum and maximum idioms. And in this case, at
1806 // least one of the comparison operands has at least one user besides
1807 // the compare (the select), which would often largely negate the
1808 // benefit of folding anyway.
1809 if (auto *CI = dyn_cast<FCmpInst>(SI->getCondition())) {
1810 if (CI->hasOneUse()) {
1811 Value *Op0 = CI->getOperand(0), *Op1 = CI->getOperand(1);
1812 if (((TV == Op0 && FV == Op1) || (FV == Op0 && TV == Op1)) &&
1813 !CI->isCommutative())
1814 return nullptr;
1815 }
1816 }
1817
1818 // Make sure that one of the select arms folds successfully.
1819 Value *NewTV = simplifyOperationIntoSelectOperand(Op, SI, /*IsTrueArm=*/true);
1820 Value *NewFV =
1821 simplifyOperationIntoSelectOperand(Op, SI, /*IsTrueArm=*/false);
1822 if (!NewTV && !NewFV)
1823 return nullptr;
1824
1825 if (SimplifyBothArms && !(NewTV && NewFV))
1826 return nullptr;
1827
1828 // Create an instruction for the arm that did not fold.
1829 if (!NewTV)
1830 NewTV = foldOperationIntoSelectOperand(Op, SI, TV, *this);
1831 if (!NewFV)
1832 NewFV = foldOperationIntoSelectOperand(Op, SI, FV, *this);
1833 return SelectInst::Create(SI->getCondition(), NewTV, NewFV, "", nullptr, SI);
1834}
1835
1837 Value *InValue, BasicBlock *InBB,
1838 const DataLayout &DL,
1839 const SimplifyQuery SQ) {
1840 // NB: It is a precondition of this transform that the operands be
1841 // phi translatable!
1843 for (Value *Op : I.operands()) {
1844 if (Op == PN)
1845 Ops.push_back(InValue);
1846 else
1847 Ops.push_back(Op->DoPHITranslation(PN->getParent(), InBB));
1848 }
1849
1850 // Don't consider the simplification successful if we get back a constant
1851 // expression. That's just an instruction in hiding.
1852 // Also reject the case where we simplify back to the phi node. We wouldn't
1853 // be able to remove it in that case.
1855 &I, Ops, SQ.getWithInstruction(InBB->getTerminator()));
1856 if (NewVal && NewVal != PN && !match(NewVal, m_ConstantExpr()))
1857 return NewVal;
1858
1859 // Check if incoming PHI value can be replaced with constant
1860 // based on implied condition.
1861 BranchInst *TerminatorBI = dyn_cast<BranchInst>(InBB->getTerminator());
1862 const ICmpInst *ICmp = dyn_cast<ICmpInst>(&I);
1863 if (TerminatorBI && TerminatorBI->isConditional() &&
1864 TerminatorBI->getSuccessor(0) != TerminatorBI->getSuccessor(1) && ICmp) {
1865 bool LHSIsTrue = TerminatorBI->getSuccessor(0) == PN->getParent();
1866 std::optional<bool> ImpliedCond = isImpliedCondition(
1867 TerminatorBI->getCondition(), ICmp->getCmpPredicate(), Ops[0], Ops[1],
1868 DL, LHSIsTrue);
1869 if (ImpliedCond)
1870 return ConstantInt::getBool(I.getType(), ImpliedCond.value());
1871 }
1872
1873 return nullptr;
1874}
1875
1877 bool AllowMultipleUses) {
1878 unsigned NumPHIValues = PN->getNumIncomingValues();
1879 if (NumPHIValues == 0)
1880 return nullptr;
1881
1882 // We normally only transform phis with a single use. However, if a PHI has
1883 // multiple uses and they are all the same operation, we can fold *all* of the
1884 // uses into the PHI.
1885 bool OneUse = PN->hasOneUse();
1886 bool IdenticalUsers = false;
1887 if (!AllowMultipleUses && !OneUse) {
1888 // Walk the use list for the instruction, comparing them to I.
1889 for (User *U : PN->users()) {
1891 if (UI != &I && !I.isIdenticalTo(UI))
1892 return nullptr;
1893 }
1894 // Otherwise, we can replace *all* users with the new PHI we form.
1895 IdenticalUsers = true;
1896 }
1897
1898 // Check that all operands are phi-translatable.
1899 for (Value *Op : I.operands()) {
1900 if (Op == PN)
1901 continue;
1902
1903 // Non-instructions never require phi-translation.
1904 auto *I = dyn_cast<Instruction>(Op);
1905 if (!I)
1906 continue;
1907
1908 // Phi-translate can handle phi nodes in the same block.
1909 if (isa<PHINode>(I))
1910 if (I->getParent() == PN->getParent())
1911 continue;
1912
1913 // Operand dominates the block, no phi-translation necessary.
1914 if (DT.dominates(I, PN->getParent()))
1915 continue;
1916
1917 // Not phi-translatable, bail out.
1918 return nullptr;
1919 }
1920
1921 // Check to see whether the instruction can be folded into each phi operand.
1922 // If there is one operand that does not fold, remember the BB it is in.
1923 SmallVector<Value *> NewPhiValues;
1924 SmallVector<unsigned int> OpsToMoveUseToIncomingBB;
1925 bool SeenNonSimplifiedInVal = false;
1926 for (unsigned i = 0; i != NumPHIValues; ++i) {
1927 Value *InVal = PN->getIncomingValue(i);
1928 BasicBlock *InBB = PN->getIncomingBlock(i);
1929
1930 if (auto *NewVal = simplifyInstructionWithPHI(I, PN, InVal, InBB, DL, SQ)) {
1931 NewPhiValues.push_back(NewVal);
1932 continue;
1933 }
1934
1935 // Handle some cases that can't be fully simplified, but where we know that
1936 // the two instructions will fold into one.
1937 auto WillFold = [&]() {
1938 if (!InVal->hasUseList() || !InVal->hasOneUser())
1939 return false;
1940
1941 // icmp of ucmp/scmp with constant will fold to icmp.
1942 const APInt *Ignored;
1943 if (isa<CmpIntrinsic>(InVal) &&
1944 match(&I, m_ICmp(m_Specific(PN), m_APInt(Ignored))))
1945 return true;
1946
1947 // icmp eq zext(bool), 0 will fold to !bool.
1948 if (isa<ZExtInst>(InVal) &&
1949 cast<ZExtInst>(InVal)->getSrcTy()->isIntOrIntVectorTy(1) &&
1950 match(&I,
1952 return true;
1953
1954 return false;
1955 };
1956
1957 if (WillFold()) {
1958 OpsToMoveUseToIncomingBB.push_back(i);
1959 NewPhiValues.push_back(nullptr);
1960 continue;
1961 }
1962
1963 if (!OneUse && !IdenticalUsers)
1964 return nullptr;
1965
1966 if (SeenNonSimplifiedInVal)
1967 return nullptr; // More than one non-simplified value.
1968 SeenNonSimplifiedInVal = true;
1969
1970 // If there is exactly one non-simplified value, we can insert a copy of the
1971 // operation in that block. However, if this is a critical edge, we would
1972 // be inserting the computation on some other paths (e.g. inside a loop).
1973 // Only do this if the pred block is unconditionally branching into the phi
1974 // block. Also, make sure that the pred block is not dead code.
1976 if (!BI || !BI->isUnconditional() || !DT.isReachableFromEntry(InBB))
1977 return nullptr;
1978
1979 NewPhiValues.push_back(nullptr);
1980 OpsToMoveUseToIncomingBB.push_back(i);
1981
1982 // Do not push the operation across a loop backedge. This could result in
1983 // an infinite combine loop, and is generally non-profitable (especially
1984 // if the operation was originally outside the loop).
1985 if (isBackEdge(InBB, PN->getParent()))
1986 return nullptr;
1987 }
1988
1989 // Clone the instruction that uses the phi node and move it into the incoming
1990 // BB because we know that the next iteration of InstCombine will simplify it.
1992 for (auto OpIndex : OpsToMoveUseToIncomingBB) {
1994 BasicBlock *OpBB = PN->getIncomingBlock(OpIndex);
1995
1996 Instruction *Clone = Clones.lookup(OpBB);
1997 if (!Clone) {
1998 Clone = I.clone();
1999 for (Use &U : Clone->operands()) {
2000 if (U == PN)
2001 U = Op;
2002 else
2003 U = U->DoPHITranslation(PN->getParent(), OpBB);
2004 }
2005 Clone = InsertNewInstBefore(Clone, OpBB->getTerminator()->getIterator());
2006 Clones.insert({OpBB, Clone});
2007 // We may have speculated the instruction.
2009 }
2010
2011 NewPhiValues[OpIndex] = Clone;
2012 }
2013
2014 // Okay, we can do the transformation: create the new PHI node.
2015 PHINode *NewPN = PHINode::Create(I.getType(), PN->getNumIncomingValues());
2016 InsertNewInstBefore(NewPN, PN->getIterator());
2017 NewPN->takeName(PN);
2018 NewPN->setDebugLoc(PN->getDebugLoc());
2019
2020 for (unsigned i = 0; i != NumPHIValues; ++i)
2021 NewPN->addIncoming(NewPhiValues[i], PN->getIncomingBlock(i));
2022
2023 if (IdenticalUsers) {
2024 // Collect and deduplicate users up-front to avoid iterator invalidation.
2026 for (User *U : PN->users()) {
2028 if (User == &I)
2029 continue;
2030 ToReplace.insert(User);
2031 }
2032 for (Instruction *I : ToReplace) {
2033 replaceInstUsesWith(*I, NewPN);
2035 }
2036 OneUse = true;
2037 }
2038
2039 if (OneUse) {
2040 replaceAllDbgUsesWith(*PN, *NewPN, *PN, DT);
2041 }
2042 return replaceInstUsesWith(I, NewPN);
2043}
2044
2046 if (!BO.isAssociative())
2047 return nullptr;
2048
2049 // Find the interleaved binary ops.
2050 auto Opc = BO.getOpcode();
2051 auto *BO0 = dyn_cast<BinaryOperator>(BO.getOperand(0));
2052 auto *BO1 = dyn_cast<BinaryOperator>(BO.getOperand(1));
2053 if (!BO0 || !BO1 || !BO0->hasNUses(2) || !BO1->hasNUses(2) ||
2054 BO0->getOpcode() != Opc || BO1->getOpcode() != Opc ||
2055 !BO0->isAssociative() || !BO1->isAssociative() ||
2056 BO0->getParent() != BO1->getParent())
2057 return nullptr;
2058
2059 assert(BO.isCommutative() && BO0->isCommutative() && BO1->isCommutative() &&
2060 "Expected commutative instructions!");
2061
2062 // Find the matching phis, forming the recurrences.
2063 PHINode *PN0, *PN1;
2064 Value *Start0, *Step0, *Start1, *Step1;
2065 if (!matchSimpleRecurrence(BO0, PN0, Start0, Step0) || !PN0->hasOneUse() ||
2066 !matchSimpleRecurrence(BO1, PN1, Start1, Step1) || !PN1->hasOneUse() ||
2067 PN0->getParent() != PN1->getParent())
2068 return nullptr;
2069
2070 assert(PN0->getNumIncomingValues() == 2 && PN1->getNumIncomingValues() == 2 &&
2071 "Expected PHIs with two incoming values!");
2072
2073 // Convert the start and step values to constants.
2074 auto *Init0 = dyn_cast<Constant>(Start0);
2075 auto *Init1 = dyn_cast<Constant>(Start1);
2076 auto *C0 = dyn_cast<Constant>(Step0);
2077 auto *C1 = dyn_cast<Constant>(Step1);
2078 if (!Init0 || !Init1 || !C0 || !C1)
2079 return nullptr;
2080
2081 // Fold the recurrence constants.
2082 auto *Init = ConstantFoldBinaryInstruction(Opc, Init0, Init1);
2083 auto *C = ConstantFoldBinaryInstruction(Opc, C0, C1);
2084 if (!Init || !C)
2085 return nullptr;
2086
2087 // Create the reduced PHI.
2088 auto *NewPN = PHINode::Create(PN0->getType(), PN0->getNumIncomingValues(),
2089 "reduced.phi");
2090
2091 // Create the new binary op.
2092 auto *NewBO = BinaryOperator::Create(Opc, NewPN, C);
2093 if (Opc == Instruction::FAdd || Opc == Instruction::FMul) {
2094 // Intersect FMF flags for FADD and FMUL.
2095 FastMathFlags Intersect = BO0->getFastMathFlags() &
2096 BO1->getFastMathFlags() & BO.getFastMathFlags();
2097 NewBO->setFastMathFlags(Intersect);
2098 } else {
2099 OverflowTracking Flags;
2100 Flags.AllKnownNonNegative = false;
2101 Flags.AllKnownNonZero = false;
2102 Flags.mergeFlags(*BO0);
2103 Flags.mergeFlags(*BO1);
2104 Flags.mergeFlags(BO);
2105 Flags.applyFlags(*NewBO);
2106 }
2107 NewBO->takeName(&BO);
2108
2109 for (unsigned I = 0, E = PN0->getNumIncomingValues(); I != E; ++I) {
2110 auto *V = PN0->getIncomingValue(I);
2111 auto *BB = PN0->getIncomingBlock(I);
2112 if (V == Init0) {
2113 assert(((PN1->getIncomingValue(0) == Init1 &&
2114 PN1->getIncomingBlock(0) == BB) ||
2115 (PN1->getIncomingValue(1) == Init1 &&
2116 PN1->getIncomingBlock(1) == BB)) &&
2117 "Invalid incoming block!");
2118 NewPN->addIncoming(Init, BB);
2119 } else if (V == BO0) {
2120 assert(((PN1->getIncomingValue(0) == BO1 &&
2121 PN1->getIncomingBlock(0) == BB) ||
2122 (PN1->getIncomingValue(1) == BO1 &&
2123 PN1->getIncomingBlock(1) == BB)) &&
2124 "Invalid incoming block!");
2125 NewPN->addIncoming(NewBO, BB);
2126 } else
2127 llvm_unreachable("Unexpected incoming value!");
2128 }
2129
2130 LLVM_DEBUG(dbgs() << " Combined " << *PN0 << "\n " << *BO0
2131 << "\n with " << *PN1 << "\n " << *BO1
2132 << '\n');
2133
2134 // Insert the new recurrence and remove the old (dead) ones.
2135 InsertNewInstWith(NewPN, PN0->getIterator());
2136 InsertNewInstWith(NewBO, BO0->getIterator());
2137
2144
2145 return replaceInstUsesWith(BO, NewBO);
2146}
2147
2149 // Attempt to fold binary operators whose operands are simple recurrences.
2150 if (auto *NewBO = foldBinopWithRecurrence(BO))
2151 return NewBO;
2152
2153 // TODO: This should be similar to the incoming values check in foldOpIntoPhi:
2154 // we are guarding against replicating the binop in >1 predecessor.
2155 // This could miss matching a phi with 2 constant incoming values.
2156 auto *Phi0 = dyn_cast<PHINode>(BO.getOperand(0));
2157 auto *Phi1 = dyn_cast<PHINode>(BO.getOperand(1));
2158 if (!Phi0 || !Phi1 || !Phi0->hasOneUse() || !Phi1->hasOneUse() ||
2159 Phi0->getNumOperands() != Phi1->getNumOperands())
2160 return nullptr;
2161
2162 // TODO: Remove the restriction for binop being in the same block as the phis.
2163 if (BO.getParent() != Phi0->getParent() ||
2164 BO.getParent() != Phi1->getParent())
2165 return nullptr;
2166
2167 // Fold if there is at least one specific constant value in phi0 or phi1's
2168 // incoming values that comes from the same block and this specific constant
2169 // value can be used to do optimization for specific binary operator.
2170 // For example:
2171 // %phi0 = phi i32 [0, %bb0], [%i, %bb1]
2172 // %phi1 = phi i32 [%j, %bb0], [0, %bb1]
2173 // %add = add i32 %phi0, %phi1
2174 // ==>
2175 // %add = phi i32 [%j, %bb0], [%i, %bb1]
2177 /*AllowRHSConstant*/ false);
2178 if (C) {
2179 SmallVector<Value *, 4> NewIncomingValues;
2180 auto CanFoldIncomingValuePair = [&](std::tuple<Use &, Use &> T) {
2181 auto &Phi0Use = std::get<0>(T);
2182 auto &Phi1Use = std::get<1>(T);
2183 if (Phi0->getIncomingBlock(Phi0Use) != Phi1->getIncomingBlock(Phi1Use))
2184 return false;
2185 Value *Phi0UseV = Phi0Use.get();
2186 Value *Phi1UseV = Phi1Use.get();
2187 if (Phi0UseV == C)
2188 NewIncomingValues.push_back(Phi1UseV);
2189 else if (Phi1UseV == C)
2190 NewIncomingValues.push_back(Phi0UseV);
2191 else
2192 return false;
2193 return true;
2194 };
2195
2196 if (all_of(zip(Phi0->operands(), Phi1->operands()),
2197 CanFoldIncomingValuePair)) {
2198 PHINode *NewPhi =
2199 PHINode::Create(Phi0->getType(), Phi0->getNumOperands());
2200 assert(NewIncomingValues.size() == Phi0->getNumOperands() &&
2201 "The number of collected incoming values should equal the number "
2202 "of the original PHINode operands!");
2203 for (unsigned I = 0; I < Phi0->getNumOperands(); I++)
2204 NewPhi->addIncoming(NewIncomingValues[I], Phi0->getIncomingBlock(I));
2205 return NewPhi;
2206 }
2207 }
2208
2209 if (Phi0->getNumOperands() != 2 || Phi1->getNumOperands() != 2)
2210 return nullptr;
2211
2212 // Match a pair of incoming constants for one of the predecessor blocks.
2213 BasicBlock *ConstBB, *OtherBB;
2214 Constant *C0, *C1;
2215 if (match(Phi0->getIncomingValue(0), m_ImmConstant(C0))) {
2216 ConstBB = Phi0->getIncomingBlock(0);
2217 OtherBB = Phi0->getIncomingBlock(1);
2218 } else if (match(Phi0->getIncomingValue(1), m_ImmConstant(C0))) {
2219 ConstBB = Phi0->getIncomingBlock(1);
2220 OtherBB = Phi0->getIncomingBlock(0);
2221 } else {
2222 return nullptr;
2223 }
2224 if (!match(Phi1->getIncomingValueForBlock(ConstBB), m_ImmConstant(C1)))
2225 return nullptr;
2226
2227 // The block that we are hoisting to must reach here unconditionally.
2228 // Otherwise, we could be speculatively executing an expensive or
2229 // non-speculative op.
2230 auto *PredBlockBranch = dyn_cast<BranchInst>(OtherBB->getTerminator());
2231 if (!PredBlockBranch || PredBlockBranch->isConditional() ||
2232 !DT.isReachableFromEntry(OtherBB))
2233 return nullptr;
2234
2235 // TODO: This check could be tightened to only apply to binops (div/rem) that
2236 // are not safe to speculatively execute. But that could allow hoisting
2237 // potentially expensive instructions (fdiv for example).
2238 for (auto BBIter = BO.getParent()->begin(); &*BBIter != &BO; ++BBIter)
2240 return nullptr;
2241
2242 // Fold constants for the predecessor block with constant incoming values.
2243 Constant *NewC = ConstantFoldBinaryOpOperands(BO.getOpcode(), C0, C1, DL);
2244 if (!NewC)
2245 return nullptr;
2246
2247 // Make a new binop in the predecessor block with the non-constant incoming
2248 // values.
2249 Builder.SetInsertPoint(PredBlockBranch);
2250 Value *NewBO = Builder.CreateBinOp(BO.getOpcode(),
2251 Phi0->getIncomingValueForBlock(OtherBB),
2252 Phi1->getIncomingValueForBlock(OtherBB));
2253 if (auto *NotFoldedNewBO = dyn_cast<BinaryOperator>(NewBO))
2254 NotFoldedNewBO->copyIRFlags(&BO);
2255
2256 // Replace the binop with a phi of the new values. The old phis are dead.
2257 PHINode *NewPhi = PHINode::Create(BO.getType(), 2);
2258 NewPhi->addIncoming(NewBO, OtherBB);
2259 NewPhi->addIncoming(NewC, ConstBB);
2260 return NewPhi;
2261}
2262
2264 if (!isa<Constant>(I.getOperand(1)))
2265 return nullptr;
2266
2267 if (auto *Sel = dyn_cast<SelectInst>(I.getOperand(0))) {
2268 if (Instruction *NewSel = FoldOpIntoSelect(I, Sel))
2269 return NewSel;
2270 } else if (auto *PN = dyn_cast<PHINode>(I.getOperand(0))) {
2271 if (Instruction *NewPhi = foldOpIntoPhi(I, PN))
2272 return NewPhi;
2273 }
2274 return nullptr;
2275}
2276
2278 // If this GEP has only 0 indices, it is the same pointer as
2279 // Src. If Src is not a trivial GEP too, don't combine
2280 // the indices.
2281 if (GEP.hasAllZeroIndices() && !Src.hasAllZeroIndices() &&
2282 !Src.hasOneUse())
2283 return false;
2284 return true;
2285}
2286
2287/// Find a constant NewC that has property:
2288/// shuffle(NewC, ShMask) = C
2289/// Returns nullptr if such a constant does not exist e.g. ShMask=<0,0> C=<1,2>
2290///
2291/// A 1-to-1 mapping is not required. Example:
2292/// ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = <poison,5,6,poison>
2294 VectorType *NewCTy) {
2295 if (isa<ScalableVectorType>(NewCTy)) {
2296 Constant *Splat = C->getSplatValue();
2297 if (!Splat)
2298 return nullptr;
2300 }
2301
2302 if (cast<FixedVectorType>(NewCTy)->getNumElements() >
2303 cast<FixedVectorType>(C->getType())->getNumElements())
2304 return nullptr;
2305
2306 unsigned NewCNumElts = cast<FixedVectorType>(NewCTy)->getNumElements();
2307 PoisonValue *PoisonScalar = PoisonValue::get(C->getType()->getScalarType());
2308 SmallVector<Constant *, 16> NewVecC(NewCNumElts, PoisonScalar);
2309 unsigned NumElts = cast<FixedVectorType>(C->getType())->getNumElements();
2310 for (unsigned I = 0; I < NumElts; ++I) {
2311 Constant *CElt = C->getAggregateElement(I);
2312 if (ShMask[I] >= 0) {
2313 assert(ShMask[I] < (int)NumElts && "Not expecting narrowing shuffle");
2314 Constant *NewCElt = NewVecC[ShMask[I]];
2315 // Bail out if:
2316 // 1. The constant vector contains a constant expression.
2317 // 2. The shuffle needs an element of the constant vector that can't
2318 // be mapped to a new constant vector.
2319 // 3. This is a widening shuffle that copies elements of V1 into the
2320 // extended elements (extending with poison is allowed).
2321 if (!CElt || (!isa<PoisonValue>(NewCElt) && NewCElt != CElt) ||
2322 I >= NewCNumElts)
2323 return nullptr;
2324 NewVecC[ShMask[I]] = CElt;
2325 }
2326 }
2327 return ConstantVector::get(NewVecC);
2328}
2329
2331 if (!isa<VectorType>(Inst.getType()))
2332 return nullptr;
2333
2334 BinaryOperator::BinaryOps Opcode = Inst.getOpcode();
2335 Value *LHS = Inst.getOperand(0), *RHS = Inst.getOperand(1);
2336 assert(cast<VectorType>(LHS->getType())->getElementCount() ==
2337 cast<VectorType>(Inst.getType())->getElementCount());
2338 assert(cast<VectorType>(RHS->getType())->getElementCount() ==
2339 cast<VectorType>(Inst.getType())->getElementCount());
2340
2341 // If both operands of the binop are vector concatenations, then perform the
2342 // narrow binop on each pair of the source operands followed by concatenation
2343 // of the results.
2344 Value *L0, *L1, *R0, *R1;
2345 ArrayRef<int> Mask;
2346 if (match(LHS, m_Shuffle(m_Value(L0), m_Value(L1), m_Mask(Mask))) &&
2347 match(RHS, m_Shuffle(m_Value(R0), m_Value(R1), m_SpecificMask(Mask))) &&
2348 LHS->hasOneUse() && RHS->hasOneUse() &&
2349 cast<ShuffleVectorInst>(LHS)->isConcat() &&
2350 cast<ShuffleVectorInst>(RHS)->isConcat()) {
2351 // This transform does not have the speculative execution constraint as
2352 // below because the shuffle is a concatenation. The new binops are
2353 // operating on exactly the same elements as the existing binop.
2354 // TODO: We could ease the mask requirement to allow different undef lanes,
2355 // but that requires an analysis of the binop-with-undef output value.
2356 Value *NewBO0 = Builder.CreateBinOp(Opcode, L0, R0);
2357 if (auto *BO = dyn_cast<BinaryOperator>(NewBO0))
2358 BO->copyIRFlags(&Inst);
2359 Value *NewBO1 = Builder.CreateBinOp(Opcode, L1, R1);
2360 if (auto *BO = dyn_cast<BinaryOperator>(NewBO1))
2361 BO->copyIRFlags(&Inst);
2362 return new ShuffleVectorInst(NewBO0, NewBO1, Mask);
2363 }
2364
2365 auto createBinOpReverse = [&](Value *X, Value *Y) {
2366 Value *V = Builder.CreateBinOp(Opcode, X, Y, Inst.getName());
2367 if (auto *BO = dyn_cast<BinaryOperator>(V))
2368 BO->copyIRFlags(&Inst);
2369 Module *M = Inst.getModule();
2371 M, Intrinsic::vector_reverse, V->getType());
2372 return CallInst::Create(F, V);
2373 };
2374
2375 // NOTE: Reverse shuffles don't require the speculative execution protection
2376 // below because they don't affect which lanes take part in the computation.
2377
2378 Value *V1, *V2;
2379 if (match(LHS, m_VecReverse(m_Value(V1)))) {
2380 // Op(rev(V1), rev(V2)) -> rev(Op(V1, V2))
2381 if (match(RHS, m_VecReverse(m_Value(V2))) &&
2382 (LHS->hasOneUse() || RHS->hasOneUse() ||
2383 (LHS == RHS && LHS->hasNUses(2))))
2384 return createBinOpReverse(V1, V2);
2385
2386 // Op(rev(V1), RHSSplat)) -> rev(Op(V1, RHSSplat))
2387 if (LHS->hasOneUse() && isSplatValue(RHS))
2388 return createBinOpReverse(V1, RHS);
2389 }
2390 // Op(LHSSplat, rev(V2)) -> rev(Op(LHSSplat, V2))
2391 else if (isSplatValue(LHS) && match(RHS, m_OneUse(m_VecReverse(m_Value(V2)))))
2392 return createBinOpReverse(LHS, V2);
2393
2394 auto createBinOpVPReverse = [&](Value *X, Value *Y, Value *EVL) {
2395 Value *V = Builder.CreateBinOp(Opcode, X, Y, Inst.getName());
2396 if (auto *BO = dyn_cast<BinaryOperator>(V))
2397 BO->copyIRFlags(&Inst);
2398
2399 ElementCount EC = cast<VectorType>(V->getType())->getElementCount();
2400 Value *AllTrueMask = Builder.CreateVectorSplat(EC, Builder.getTrue());
2401 Module *M = Inst.getModule();
2403 M, Intrinsic::experimental_vp_reverse, V->getType());
2404 return CallInst::Create(F, {V, AllTrueMask, EVL});
2405 };
2406
2407 Value *EVL;
2409 m_Value(V1), m_AllOnes(), m_Value(EVL)))) {
2410 // Op(rev(V1), rev(V2)) -> rev(Op(V1, V2))
2412 m_Value(V2), m_AllOnes(), m_Specific(EVL))) &&
2413 (LHS->hasOneUse() || RHS->hasOneUse() ||
2414 (LHS == RHS && LHS->hasNUses(2))))
2415 return createBinOpVPReverse(V1, V2, EVL);
2416
2417 // Op(rev(V1), RHSSplat)) -> rev(Op(V1, RHSSplat))
2418 if (LHS->hasOneUse() && isSplatValue(RHS))
2419 return createBinOpVPReverse(V1, RHS, EVL);
2420 }
2421 // Op(LHSSplat, rev(V2)) -> rev(Op(LHSSplat, V2))
2422 else if (isSplatValue(LHS) &&
2424 m_Value(V2), m_AllOnes(), m_Value(EVL))))
2425 return createBinOpVPReverse(LHS, V2, EVL);
2426
2427 // It may not be safe to reorder shuffles and things like div, urem, etc.
2428 // because we may trap when executing those ops on unknown vector elements.
2429 // See PR20059.
2431 return nullptr;
2432
2433 auto createBinOpShuffle = [&](Value *X, Value *Y, ArrayRef<int> M) {
2434 Value *XY = Builder.CreateBinOp(Opcode, X, Y);
2435 if (auto *BO = dyn_cast<BinaryOperator>(XY))
2436 BO->copyIRFlags(&Inst);
2437 return new ShuffleVectorInst(XY, M);
2438 };
2439
2440 // If both arguments of the binary operation are shuffles that use the same
2441 // mask and shuffle within a single vector, move the shuffle after the binop.
2442 if (match(LHS, m_Shuffle(m_Value(V1), m_Poison(), m_Mask(Mask))) &&
2443 match(RHS, m_Shuffle(m_Value(V2), m_Poison(), m_SpecificMask(Mask))) &&
2444 V1->getType() == V2->getType() &&
2445 (LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) {
2446 // Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask)
2447 return createBinOpShuffle(V1, V2, Mask);
2448 }
2449
2450 // If both arguments of a commutative binop are select-shuffles that use the
2451 // same mask with commuted operands, the shuffles are unnecessary.
2452 if (Inst.isCommutative() &&
2453 match(LHS, m_Shuffle(m_Value(V1), m_Value(V2), m_Mask(Mask))) &&
2454 match(RHS,
2455 m_Shuffle(m_Specific(V2), m_Specific(V1), m_SpecificMask(Mask)))) {
2456 auto *LShuf = cast<ShuffleVectorInst>(LHS);
2457 auto *RShuf = cast<ShuffleVectorInst>(RHS);
2458 // TODO: Allow shuffles that contain undefs in the mask?
2459 // That is legal, but it reduces undef knowledge.
2460 // TODO: Allow arbitrary shuffles by shuffling after binop?
2461 // That might be legal, but we have to deal with poison.
2462 if (LShuf->isSelect() &&
2463 !is_contained(LShuf->getShuffleMask(), PoisonMaskElem) &&
2464 RShuf->isSelect() &&
2465 !is_contained(RShuf->getShuffleMask(), PoisonMaskElem)) {
2466 // Example:
2467 // LHS = shuffle V1, V2, <0, 5, 6, 3>
2468 // RHS = shuffle V2, V1, <0, 5, 6, 3>
2469 // LHS + RHS --> (V10+V20, V21+V11, V22+V12, V13+V23) --> V1 + V2
2470 Instruction *NewBO = BinaryOperator::Create(Opcode, V1, V2);
2471 NewBO->copyIRFlags(&Inst);
2472 return NewBO;
2473 }
2474 }
2475
2476 // If one argument is a shuffle within one vector and the other is a constant,
2477 // try moving the shuffle after the binary operation. This canonicalization
2478 // intends to move shuffles closer to other shuffles and binops closer to
2479 // other binops, so they can be folded. It may also enable demanded elements
2480 // transforms.
2481 Constant *C;
2483 m_Mask(Mask))),
2484 m_ImmConstant(C)))) {
2485 assert(Inst.getType()->getScalarType() == V1->getType()->getScalarType() &&
2486 "Shuffle should not change scalar type");
2487
2488 bool ConstOp1 = isa<Constant>(RHS);
2489 if (Constant *NewC =
2491 // For fixed vectors, lanes of NewC not used by the shuffle will be poison
2492 // which will cause UB for div/rem. Mask them with a safe constant.
2493 if (isa<FixedVectorType>(V1->getType()) && Inst.isIntDivRem())
2494 NewC = getSafeVectorConstantForBinop(Opcode, NewC, ConstOp1);
2495
2496 // Op(shuffle(V1, Mask), C) -> shuffle(Op(V1, NewC), Mask)
2497 // Op(C, shuffle(V1, Mask)) -> shuffle(Op(NewC, V1), Mask)
2498 Value *NewLHS = ConstOp1 ? V1 : NewC;
2499 Value *NewRHS = ConstOp1 ? NewC : V1;
2500 return createBinOpShuffle(NewLHS, NewRHS, Mask);
2501 }
2502 }
2503
2504 // Try to reassociate to sink a splat shuffle after a binary operation.
2505 if (Inst.isAssociative() && Inst.isCommutative()) {
2506 // Canonicalize shuffle operand as LHS.
2507 if (isa<ShuffleVectorInst>(RHS))
2508 std::swap(LHS, RHS);
2509
2510 Value *X;
2511 ArrayRef<int> MaskC;
2512 int SplatIndex;
2513 Value *Y, *OtherOp;
2514 if (!match(LHS,
2515 m_OneUse(m_Shuffle(m_Value(X), m_Undef(), m_Mask(MaskC)))) ||
2516 !match(MaskC, m_SplatOrPoisonMask(SplatIndex)) ||
2517 X->getType() != Inst.getType() ||
2518 !match(RHS, m_OneUse(m_BinOp(Opcode, m_Value(Y), m_Value(OtherOp)))))
2519 return nullptr;
2520
2521 // FIXME: This may not be safe if the analysis allows undef elements. By
2522 // moving 'Y' before the splat shuffle, we are implicitly assuming
2523 // that it is not undef/poison at the splat index.
2524 if (isSplatValue(OtherOp, SplatIndex)) {
2525 std::swap(Y, OtherOp);
2526 } else if (!isSplatValue(Y, SplatIndex)) {
2527 return nullptr;
2528 }
2529
2530 // X and Y are splatted values, so perform the binary operation on those
2531 // values followed by a splat followed by the 2nd binary operation:
2532 // bo (splat X), (bo Y, OtherOp) --> bo (splat (bo X, Y)), OtherOp
2533 Value *NewBO = Builder.CreateBinOp(Opcode, X, Y);
2534 SmallVector<int, 8> NewMask(MaskC.size(), SplatIndex);
2535 Value *NewSplat = Builder.CreateShuffleVector(NewBO, NewMask);
2536 Instruction *R = BinaryOperator::Create(Opcode, NewSplat, OtherOp);
2537
2538 // Intersect FMF on both new binops. Other (poison-generating) flags are
2539 // dropped to be safe.
2540 if (isa<FPMathOperator>(R)) {
2541 R->copyFastMathFlags(&Inst);
2542 R->andIRFlags(RHS);
2543 }
2544 if (auto *NewInstBO = dyn_cast<BinaryOperator>(NewBO))
2545 NewInstBO->copyIRFlags(R);
2546 return R;
2547 }
2548
2549 return nullptr;
2550}
2551
2552/// Try to narrow the width of a binop if at least 1 operand is an extend of
2553/// of a value. This requires a potentially expensive known bits check to make
2554/// sure the narrow op does not overflow.
2555Instruction *InstCombinerImpl::narrowMathIfNoOverflow(BinaryOperator &BO) {
2556 // We need at least one extended operand.
2557 Value *Op0 = BO.getOperand(0), *Op1 = BO.getOperand(1);
2558
2559 // If this is a sub, we swap the operands since we always want an extension
2560 // on the RHS. The LHS can be an extension or a constant.
2561 if (BO.getOpcode() == Instruction::Sub)
2562 std::swap(Op0, Op1);
2563
2564 Value *X;
2565 bool IsSext = match(Op0, m_SExt(m_Value(X)));
2566 if (!IsSext && !match(Op0, m_ZExt(m_Value(X))))
2567 return nullptr;
2568
2569 // If both operands are the same extension from the same source type and we
2570 // can eliminate at least one (hasOneUse), this might work.
2571 CastInst::CastOps CastOpc = IsSext ? Instruction::SExt : Instruction::ZExt;
2572 Value *Y;
2573 if (!(match(Op1, m_ZExtOrSExt(m_Value(Y))) && X->getType() == Y->getType() &&
2574 cast<Operator>(Op1)->getOpcode() == CastOpc &&
2575 (Op0->hasOneUse() || Op1->hasOneUse()))) {
2576 // If that did not match, see if we have a suitable constant operand.
2577 // Truncating and extending must produce the same constant.
2578 Constant *WideC;
2579 if (!Op0->hasOneUse() || !match(Op1, m_Constant(WideC)))
2580 return nullptr;
2581 Constant *NarrowC = getLosslessInvCast(WideC, X->getType(), CastOpc, DL);
2582 if (!NarrowC)
2583 return nullptr;
2584 Y = NarrowC;
2585 }
2586
2587 // Swap back now that we found our operands.
2588 if (BO.getOpcode() == Instruction::Sub)
2589 std::swap(X, Y);
2590
2591 // Both operands have narrow versions. Last step: the math must not overflow
2592 // in the narrow width.
2593 if (!willNotOverflow(BO.getOpcode(), X, Y, BO, IsSext))
2594 return nullptr;
2595
2596 // bo (ext X), (ext Y) --> ext (bo X, Y)
2597 // bo (ext X), C --> ext (bo X, C')
2598 Value *NarrowBO = Builder.CreateBinOp(BO.getOpcode(), X, Y, "narrow");
2599 if (auto *NewBinOp = dyn_cast<BinaryOperator>(NarrowBO)) {
2600 if (IsSext)
2601 NewBinOp->setHasNoSignedWrap();
2602 else
2603 NewBinOp->setHasNoUnsignedWrap();
2604 }
2605 return CastInst::Create(CastOpc, NarrowBO, BO.getType());
2606}
2607
2608/// Determine nowrap flags for (gep (gep p, x), y) to (gep p, (x + y))
2609/// transform.
2614
2615/// Thread a GEP operation with constant indices through the constant true/false
2616/// arms of a select.
2618 InstCombiner::BuilderTy &Builder) {
2619 if (!GEP.hasAllConstantIndices())
2620 return nullptr;
2621
2622 Instruction *Sel;
2623 Value *Cond;
2624 Constant *TrueC, *FalseC;
2625 if (!match(GEP.getPointerOperand(), m_Instruction(Sel)) ||
2626 !match(Sel,
2627 m_Select(m_Value(Cond), m_Constant(TrueC), m_Constant(FalseC))))
2628 return nullptr;
2629
2630 // gep (select Cond, TrueC, FalseC), IndexC --> select Cond, TrueC', FalseC'
2631 // Propagate 'inbounds' and metadata from existing instructions.
2632 // Note: using IRBuilder to create the constants for efficiency.
2633 SmallVector<Value *, 4> IndexC(GEP.indices());
2634 GEPNoWrapFlags NW = GEP.getNoWrapFlags();
2635 Type *Ty = GEP.getSourceElementType();
2636 Value *NewTrueC = Builder.CreateGEP(Ty, TrueC, IndexC, "", NW);
2637 Value *NewFalseC = Builder.CreateGEP(Ty, FalseC, IndexC, "", NW);
2638 return SelectInst::Create(Cond, NewTrueC, NewFalseC, "", nullptr, Sel);
2639}
2640
2641// Canonicalization:
2642// gep T, (gep i8, base, C1), (Index + C2) into
2643// gep T, (gep i8, base, C1 + C2 * sizeof(T)), Index
2645 GEPOperator *Src,
2646 InstCombinerImpl &IC) {
2647 if (GEP.getNumIndices() != 1)
2648 return nullptr;
2649 auto &DL = IC.getDataLayout();
2650 Value *Base;
2651 const APInt *C1;
2652 if (!match(Src, m_PtrAdd(m_Value(Base), m_APInt(C1))))
2653 return nullptr;
2654 Value *VarIndex;
2655 const APInt *C2;
2656 Type *PtrTy = Src->getType()->getScalarType();
2657 unsigned IndexSizeInBits = DL.getIndexTypeSizeInBits(PtrTy);
2658 if (!match(GEP.getOperand(1), m_AddLike(m_Value(VarIndex), m_APInt(C2))))
2659 return nullptr;
2660 if (C1->getBitWidth() != IndexSizeInBits ||
2661 C2->getBitWidth() != IndexSizeInBits)
2662 return nullptr;
2663 Type *BaseType = GEP.getSourceElementType();
2665 return nullptr;
2666 APInt TypeSize(IndexSizeInBits, DL.getTypeAllocSize(BaseType));
2667 APInt NewOffset = TypeSize * *C2 + *C1;
2668 if (NewOffset.isZero() ||
2669 (Src->hasOneUse() && GEP.getOperand(1)->hasOneUse())) {
2671 if (GEP.hasNoUnsignedWrap() &&
2672 cast<GEPOperator>(Src)->hasNoUnsignedWrap() &&
2673 match(GEP.getOperand(1), m_NUWAddLike(m_Value(), m_Value()))) {
2675 if (GEP.isInBounds() && cast<GEPOperator>(Src)->isInBounds())
2676 Flags |= GEPNoWrapFlags::inBounds();
2677 }
2678
2679 Value *GEPConst =
2680 IC.Builder.CreatePtrAdd(Base, IC.Builder.getInt(NewOffset), "", Flags);
2681 return GetElementPtrInst::Create(BaseType, GEPConst, VarIndex, Flags);
2682 }
2683
2684 return nullptr;
2685}
2686
2687/// Combine constant offsets separated by variable offsets.
2688/// ptradd (ptradd (ptradd p, C1), x), C2 -> ptradd (ptradd p, x), C1+C2
2690 InstCombinerImpl &IC) {
2691 if (!GEP.hasAllConstantIndices())
2692 return nullptr;
2693
2696 auto *InnerGEP = dyn_cast<GetElementPtrInst>(GEP.getPointerOperand());
2697 while (true) {
2698 if (!InnerGEP)
2699 return nullptr;
2700
2701 NW = NW.intersectForReassociate(InnerGEP->getNoWrapFlags());
2702 if (InnerGEP->hasAllConstantIndices())
2703 break;
2704
2705 if (!InnerGEP->hasOneUse())
2706 return nullptr;
2707
2708 Skipped.push_back(InnerGEP);
2709 InnerGEP = dyn_cast<GetElementPtrInst>(InnerGEP->getPointerOperand());
2710 }
2711
2712 // The two constant offset GEPs are directly adjacent: Let normal offset
2713 // merging handle it.
2714 if (Skipped.empty())
2715 return nullptr;
2716
2717 // FIXME: This one-use check is not strictly necessary. Consider relaxing it
2718 // if profitable.
2719 if (!InnerGEP->hasOneUse())
2720 return nullptr;
2721
2722 // Don't bother with vector splats.
2723 Type *Ty = GEP.getType();
2724 if (InnerGEP->getType() != Ty)
2725 return nullptr;
2726
2727 const DataLayout &DL = IC.getDataLayout();
2728 APInt Offset(DL.getIndexTypeSizeInBits(Ty), 0);
2729 if (!GEP.accumulateConstantOffset(DL, Offset) ||
2730 !InnerGEP->accumulateConstantOffset(DL, Offset))
2731 return nullptr;
2732
2733 IC.replaceOperand(*Skipped.back(), 0, InnerGEP->getPointerOperand());
2734 for (GetElementPtrInst *SkippedGEP : Skipped)
2735 SkippedGEP->setNoWrapFlags(NW);
2736
2737 return IC.replaceInstUsesWith(
2738 GEP,
2739 IC.Builder.CreatePtrAdd(Skipped.front(), IC.Builder.getInt(Offset), "",
2740 NW.intersectForOffsetAdd(GEP.getNoWrapFlags())));
2741}
2742
2744 GEPOperator *Src) {
2745 // Combine Indices - If the source pointer to this getelementptr instruction
2746 // is a getelementptr instruction with matching element type, combine the
2747 // indices of the two getelementptr instructions into a single instruction.
2748 if (!shouldMergeGEPs(*cast<GEPOperator>(&GEP), *Src))
2749 return nullptr;
2750
2751 if (auto *I = canonicalizeGEPOfConstGEPI8(GEP, Src, *this))
2752 return I;
2753
2754 if (auto *I = combineConstantOffsets(GEP, *this))
2755 return I;
2756
2757 if (Src->getResultElementType() != GEP.getSourceElementType())
2758 return nullptr;
2759
2760 // Find out whether the last index in the source GEP is a sequential idx.
2761 bool EndsWithSequential = false;
2762 for (gep_type_iterator I = gep_type_begin(*Src), E = gep_type_end(*Src);
2763 I != E; ++I)
2764 EndsWithSequential = I.isSequential();
2765 if (!EndsWithSequential)
2766 return nullptr;
2767
2768 // Replace: gep (gep %P, long B), long A, ...
2769 // With: T = long A+B; gep %P, T, ...
2770 Value *SO1 = Src->getOperand(Src->getNumOperands() - 1);
2771 Value *GO1 = GEP.getOperand(1);
2772
2773 // If they aren't the same type, then the input hasn't been processed
2774 // by the loop above yet (which canonicalizes sequential index types to
2775 // intptr_t). Just avoid transforming this until the input has been
2776 // normalized.
2777 if (SO1->getType() != GO1->getType())
2778 return nullptr;
2779
2780 Value *Sum =
2781 simplifyAddInst(GO1, SO1, false, false, SQ.getWithInstruction(&GEP));
2782 // Only do the combine when we are sure the cost after the
2783 // merge is never more than that before the merge.
2784 if (Sum == nullptr)
2785 return nullptr;
2786
2788 Indices.append(Src->op_begin() + 1, Src->op_end() - 1);
2789 Indices.push_back(Sum);
2790 Indices.append(GEP.op_begin() + 2, GEP.op_end());
2791
2792 // Don't create GEPs with more than one non-zero index.
2793 unsigned NumNonZeroIndices = count_if(Indices, [](Value *Idx) {
2794 auto *C = dyn_cast<Constant>(Idx);
2795 return !C || !C->isNullValue();
2796 });
2797 if (NumNonZeroIndices > 1)
2798 return nullptr;
2799
2800 return replaceInstUsesWith(
2801 GEP, Builder.CreateGEP(
2802 Src->getSourceElementType(), Src->getOperand(0), Indices, "",
2804}
2805
2808 bool &DoesConsume, unsigned Depth) {
2809 static Value *const NonNull = reinterpret_cast<Value *>(uintptr_t(1));
2810 // ~(~(X)) -> X.
2811 Value *A, *B;
2812 if (match(V, m_Not(m_Value(A)))) {
2813 DoesConsume = true;
2814 return A;
2815 }
2816
2817 Constant *C;
2818 // Constants can be considered to be not'ed values.
2819 if (match(V, m_ImmConstant(C)))
2820 return ConstantExpr::getNot(C);
2821
2823 return nullptr;
2824
2825 // The rest of the cases require that we invert all uses so don't bother
2826 // doing the analysis if we know we can't use the result.
2827 if (!WillInvertAllUses)
2828 return nullptr;
2829
2830 // Compares can be inverted if all of their uses are being modified to use
2831 // the ~V.
2832 if (auto *I = dyn_cast<CmpInst>(V)) {
2833 if (Builder != nullptr)
2834 return Builder->CreateCmp(I->getInversePredicate(), I->getOperand(0),
2835 I->getOperand(1));
2836 return NonNull;
2837 }
2838
2839 // If `V` is of the form `A + B` then `-1 - V` can be folded into
2840 // `(-1 - B) - A` if we are willing to invert all of the uses.
2841 if (match(V, m_Add(m_Value(A), m_Value(B)))) {
2842 if (auto *BV = getFreelyInvertedImpl(B, B->hasOneUse(), Builder,
2843 DoesConsume, Depth))
2844 return Builder ? Builder->CreateSub(BV, A) : NonNull;
2845 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2846 DoesConsume, Depth))
2847 return Builder ? Builder->CreateSub(AV, B) : NonNull;
2848 return nullptr;
2849 }
2850
2851 // If `V` is of the form `A ^ ~B` then `~(A ^ ~B)` can be folded
2852 // into `A ^ B` if we are willing to invert all of the uses.
2853 if (match(V, m_Xor(m_Value(A), m_Value(B)))) {
2854 if (auto *BV = getFreelyInvertedImpl(B, B->hasOneUse(), Builder,
2855 DoesConsume, Depth))
2856 return Builder ? Builder->CreateXor(A, BV) : NonNull;
2857 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2858 DoesConsume, Depth))
2859 return Builder ? Builder->CreateXor(AV, B) : NonNull;
2860 return nullptr;
2861 }
2862
2863 // If `V` is of the form `B - A` then `-1 - V` can be folded into
2864 // `A + (-1 - B)` if we are willing to invert all of the uses.
2865 if (match(V, m_Sub(m_Value(A), m_Value(B)))) {
2866 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2867 DoesConsume, Depth))
2868 return Builder ? Builder->CreateAdd(AV, B) : NonNull;
2869 return nullptr;
2870 }
2871
2872 // If `V` is of the form `(~A) s>> B` then `~((~A) s>> B)` can be folded
2873 // into `A s>> B` if we are willing to invert all of the uses.
2874 if (match(V, m_AShr(m_Value(A), m_Value(B)))) {
2875 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2876 DoesConsume, Depth))
2877 return Builder ? Builder->CreateAShr(AV, B) : NonNull;
2878 return nullptr;
2879 }
2880
2881 Value *Cond;
2882 // LogicOps are special in that we canonicalize them at the cost of an
2883 // instruction.
2884 bool IsSelect = match(V, m_Select(m_Value(Cond), m_Value(A), m_Value(B))) &&
2886 // Selects/min/max with invertible operands are freely invertible
2887 if (IsSelect || match(V, m_MaxOrMin(m_Value(A), m_Value(B)))) {
2888 bool LocalDoesConsume = DoesConsume;
2889 if (!getFreelyInvertedImpl(B, B->hasOneUse(), /*Builder*/ nullptr,
2890 LocalDoesConsume, Depth))
2891 return nullptr;
2892 if (Value *NotA = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2893 LocalDoesConsume, Depth)) {
2894 DoesConsume = LocalDoesConsume;
2895 if (Builder != nullptr) {
2896 Value *NotB = getFreelyInvertedImpl(B, B->hasOneUse(), Builder,
2897 DoesConsume, Depth);
2898 assert(NotB != nullptr &&
2899 "Unable to build inverted value for known freely invertable op");
2900 if (auto *II = dyn_cast<IntrinsicInst>(V))
2901 return Builder->CreateBinaryIntrinsic(
2902 getInverseMinMaxIntrinsic(II->getIntrinsicID()), NotA, NotB);
2903 return Builder->CreateSelect(Cond, NotA, NotB);
2904 }
2905 return NonNull;
2906 }
2907 }
2908
2909 if (PHINode *PN = dyn_cast<PHINode>(V)) {
2910 bool LocalDoesConsume = DoesConsume;
2912 for (Use &U : PN->operands()) {
2913 BasicBlock *IncomingBlock = PN->getIncomingBlock(U);
2914 Value *NewIncomingVal = getFreelyInvertedImpl(
2915 U.get(), /*WillInvertAllUses=*/false,
2916 /*Builder=*/nullptr, LocalDoesConsume, MaxAnalysisRecursionDepth - 1);
2917 if (NewIncomingVal == nullptr)
2918 return nullptr;
2919 // Make sure that we can safely erase the original PHI node.
2920 if (NewIncomingVal == V)
2921 return nullptr;
2922 if (Builder != nullptr)
2923 IncomingValues.emplace_back(NewIncomingVal, IncomingBlock);
2924 }
2925
2926 DoesConsume = LocalDoesConsume;
2927 if (Builder != nullptr) {
2929 Builder->SetInsertPoint(PN);
2930 PHINode *NewPN =
2931 Builder->CreatePHI(PN->getType(), PN->getNumIncomingValues());
2932 for (auto [Val, Pred] : IncomingValues)
2933 NewPN->addIncoming(Val, Pred);
2934 return NewPN;
2935 }
2936 return NonNull;
2937 }
2938
2939 if (match(V, m_SExtLike(m_Value(A)))) {
2940 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2941 DoesConsume, Depth))
2942 return Builder ? Builder->CreateSExt(AV, V->getType()) : NonNull;
2943 return nullptr;
2944 }
2945
2946 if (match(V, m_Trunc(m_Value(A)))) {
2947 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2948 DoesConsume, Depth))
2949 return Builder ? Builder->CreateTrunc(AV, V->getType()) : NonNull;
2950 return nullptr;
2951 }
2952
2953 // De Morgan's Laws:
2954 // (~(A | B)) -> (~A & ~B)
2955 // (~(A & B)) -> (~A | ~B)
2956 auto TryInvertAndOrUsingDeMorgan = [&](Instruction::BinaryOps Opcode,
2957 bool IsLogical, Value *A,
2958 Value *B) -> Value * {
2959 bool LocalDoesConsume = DoesConsume;
2960 if (!getFreelyInvertedImpl(B, B->hasOneUse(), /*Builder=*/nullptr,
2961 LocalDoesConsume, Depth))
2962 return nullptr;
2963 if (auto *NotA = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2964 LocalDoesConsume, Depth)) {
2965 auto *NotB = getFreelyInvertedImpl(B, B->hasOneUse(), Builder,
2966 LocalDoesConsume, Depth);
2967 DoesConsume = LocalDoesConsume;
2968 if (IsLogical)
2969 return Builder ? Builder->CreateLogicalOp(Opcode, NotA, NotB) : NonNull;
2970 return Builder ? Builder->CreateBinOp(Opcode, NotA, NotB) : NonNull;
2971 }
2972
2973 return nullptr;
2974 };
2975
2976 if (match(V, m_Or(m_Value(A), m_Value(B))))
2977 return TryInvertAndOrUsingDeMorgan(Instruction::And, /*IsLogical=*/false, A,
2978 B);
2979
2980 if (match(V, m_And(m_Value(A), m_Value(B))))
2981 return TryInvertAndOrUsingDeMorgan(Instruction::Or, /*IsLogical=*/false, A,
2982 B);
2983
2984 if (match(V, m_LogicalOr(m_Value(A), m_Value(B))))
2985 return TryInvertAndOrUsingDeMorgan(Instruction::And, /*IsLogical=*/true, A,
2986 B);
2987
2988 if (match(V, m_LogicalAnd(m_Value(A), m_Value(B))))
2989 return TryInvertAndOrUsingDeMorgan(Instruction::Or, /*IsLogical=*/true, A,
2990 B);
2991
2992 return nullptr;
2993}
2994
2995/// Return true if we should canonicalize the gep to an i8 ptradd.
2997 Value *PtrOp = GEP.getOperand(0);
2998 Type *GEPEltType = GEP.getSourceElementType();
2999 if (GEPEltType->isIntegerTy(8))
3000 return false;
3001
3002 // Canonicalize scalable GEPs to an explicit offset using the llvm.vscale
3003 // intrinsic. This has better support in BasicAA.
3004 if (GEPEltType->isScalableTy())
3005 return true;
3006
3007 // gep i32 p, mul(O, C) -> gep i8, p, mul(O, C*4) to fold the two multiplies
3008 // together.
3009 if (GEP.getNumIndices() == 1 &&
3010 match(GEP.getOperand(1),
3012 m_Shl(m_Value(), m_ConstantInt())))))
3013 return true;
3014
3015 // gep (gep %p, C1), %x, C2 is expanded so the two constants can
3016 // possibly be merged together.
3017 auto PtrOpGep = dyn_cast<GEPOperator>(PtrOp);
3018 return PtrOpGep && PtrOpGep->hasAllConstantIndices() &&
3019 any_of(GEP.indices(), [](Value *V) {
3020 const APInt *C;
3021 return match(V, m_APInt(C)) && !C->isZero();
3022 });
3023}
3024
3026 IRBuilderBase &Builder) {
3027 auto *Op1 = dyn_cast<GetElementPtrInst>(PN->getOperand(0));
3028 if (!Op1)
3029 return nullptr;
3030
3031 // Don't fold a GEP into itself through a PHI node. This can only happen
3032 // through the back-edge of a loop. Folding a GEP into itself means that
3033 // the value of the previous iteration needs to be stored in the meantime,
3034 // thus requiring an additional register variable to be live, but not
3035 // actually achieving anything (the GEP still needs to be executed once per
3036 // loop iteration).
3037 if (Op1 == &GEP)
3038 return nullptr;
3039 GEPNoWrapFlags NW = Op1->getNoWrapFlags();
3040
3041 int DI = -1;
3042
3043 for (auto I = PN->op_begin()+1, E = PN->op_end(); I !=E; ++I) {
3044 auto *Op2 = dyn_cast<GetElementPtrInst>(*I);
3045 if (!Op2 || Op1->getNumOperands() != Op2->getNumOperands() ||
3046 Op1->getSourceElementType() != Op2->getSourceElementType())
3047 return nullptr;
3048
3049 // As for Op1 above, don't try to fold a GEP into itself.
3050 if (Op2 == &GEP)
3051 return nullptr;
3052
3053 // Keep track of the type as we walk the GEP.
3054 Type *CurTy = nullptr;
3055
3056 for (unsigned J = 0, F = Op1->getNumOperands(); J != F; ++J) {
3057 if (Op1->getOperand(J)->getType() != Op2->getOperand(J)->getType())
3058 return nullptr;
3059
3060 if (Op1->getOperand(J) != Op2->getOperand(J)) {
3061 if (DI == -1) {
3062 // We have not seen any differences yet in the GEPs feeding the
3063 // PHI yet, so we record this one if it is allowed to be a
3064 // variable.
3065
3066 // The first two arguments can vary for any GEP, the rest have to be
3067 // static for struct slots
3068 if (J > 1) {
3069 assert(CurTy && "No current type?");
3070 if (CurTy->isStructTy())
3071 return nullptr;
3072 }
3073
3074 DI = J;
3075 } else {
3076 // The GEP is different by more than one input. While this could be
3077 // extended to support GEPs that vary by more than one variable it
3078 // doesn't make sense since it greatly increases the complexity and
3079 // would result in an R+R+R addressing mode which no backend
3080 // directly supports and would need to be broken into several
3081 // simpler instructions anyway.
3082 return nullptr;
3083 }
3084 }
3085
3086 // Sink down a layer of the type for the next iteration.
3087 if (J > 0) {
3088 if (J == 1) {
3089 CurTy = Op1->getSourceElementType();
3090 } else {
3091 CurTy =
3092 GetElementPtrInst::getTypeAtIndex(CurTy, Op1->getOperand(J));
3093 }
3094 }
3095 }
3096
3097 NW &= Op2->getNoWrapFlags();
3098 }
3099
3100 // If not all GEPs are identical we'll have to create a new PHI node.
3101 // Check that the old PHI node has only one use so that it will get
3102 // removed.
3103 if (DI != -1 && !PN->hasOneUse())
3104 return nullptr;
3105
3106 auto *NewGEP = cast<GetElementPtrInst>(Op1->clone());
3107 NewGEP->setNoWrapFlags(NW);
3108
3109 if (DI == -1) {
3110 // All the GEPs feeding the PHI are identical. Clone one down into our
3111 // BB so that it can be merged with the current GEP.
3112 } else {
3113 // All the GEPs feeding the PHI differ at a single offset. Clone a GEP
3114 // into the current block so it can be merged, and create a new PHI to
3115 // set that index.
3116 PHINode *NewPN;
3117 {
3118 IRBuilderBase::InsertPointGuard Guard(Builder);
3119 Builder.SetInsertPoint(PN);
3120 NewPN = Builder.CreatePHI(Op1->getOperand(DI)->getType(),
3121 PN->getNumOperands());
3122 }
3123
3124 for (auto &I : PN->operands())
3125 NewPN->addIncoming(cast<GEPOperator>(I)->getOperand(DI),
3126 PN->getIncomingBlock(I));
3127
3128 NewGEP->setOperand(DI, NewPN);
3129 }
3130
3131 NewGEP->insertBefore(*GEP.getParent(), GEP.getParent()->getFirstInsertionPt());
3132 return NewGEP;
3133}
3134
3136 Value *PtrOp = GEP.getOperand(0);
3137 SmallVector<Value *, 8> Indices(GEP.indices());
3138 Type *GEPType = GEP.getType();
3139 Type *GEPEltType = GEP.getSourceElementType();
3140 if (Value *V =
3141 simplifyGEPInst(GEPEltType, PtrOp, Indices, GEP.getNoWrapFlags(),
3142 SQ.getWithInstruction(&GEP)))
3143 return replaceInstUsesWith(GEP, V);
3144
3145 // For vector geps, use the generic demanded vector support.
3146 // Skip if GEP return type is scalable. The number of elements is unknown at
3147 // compile-time.
3148 if (auto *GEPFVTy = dyn_cast<FixedVectorType>(GEPType)) {
3149 auto VWidth = GEPFVTy->getNumElements();
3150 APInt PoisonElts(VWidth, 0);
3151 APInt AllOnesEltMask(APInt::getAllOnes(VWidth));
3152 if (Value *V = SimplifyDemandedVectorElts(&GEP, AllOnesEltMask,
3153 PoisonElts)) {
3154 if (V != &GEP)
3155 return replaceInstUsesWith(GEP, V);
3156 return &GEP;
3157 }
3158 }
3159
3160 // Eliminate unneeded casts for indices, and replace indices which displace
3161 // by multiples of a zero size type with zero.
3162 bool MadeChange = false;
3163
3164 // Index width may not be the same width as pointer width.
3165 // Data layout chooses the right type based on supported integer types.
3166 Type *NewScalarIndexTy =
3167 DL.getIndexType(GEP.getPointerOperandType()->getScalarType());
3168
3170 for (User::op_iterator I = GEP.op_begin() + 1, E = GEP.op_end(); I != E;
3171 ++I, ++GTI) {
3172 // Skip indices into struct types.
3173 if (GTI.isStruct())
3174 continue;
3175
3176 Type *IndexTy = (*I)->getType();
3177 Type *NewIndexType =
3178 IndexTy->isVectorTy()
3179 ? VectorType::get(NewScalarIndexTy,
3180 cast<VectorType>(IndexTy)->getElementCount())
3181 : NewScalarIndexTy;
3182
3183 // If the element type has zero size then any index over it is equivalent
3184 // to an index of zero, so replace it with zero if it is not zero already.
3185 Type *EltTy = GTI.getIndexedType();
3186 if (EltTy->isSized() && DL.getTypeAllocSize(EltTy).isZero())
3187 if (!isa<Constant>(*I) || !match(I->get(), m_Zero())) {
3188 *I = Constant::getNullValue(NewIndexType);
3189 MadeChange = true;
3190 }
3191
3192 if (IndexTy != NewIndexType) {
3193 // If we are using a wider index than needed for this platform, shrink
3194 // it to what we need. If narrower, sign-extend it to what we need.
3195 // This explicit cast can make subsequent optimizations more obvious.
3196 if (IndexTy->getScalarSizeInBits() <
3197 NewIndexType->getScalarSizeInBits()) {
3198 if (GEP.hasNoUnsignedWrap() && GEP.hasNoUnsignedSignedWrap())
3199 *I = Builder.CreateZExt(*I, NewIndexType, "", /*IsNonNeg=*/true);
3200 else
3201 *I = Builder.CreateSExt(*I, NewIndexType);
3202 } else {
3203 *I = Builder.CreateTrunc(*I, NewIndexType, "", GEP.hasNoUnsignedWrap(),
3204 GEP.hasNoUnsignedSignedWrap());
3205 }
3206 MadeChange = true;
3207 }
3208 }
3209 if (MadeChange)
3210 return &GEP;
3211
3212 // Canonicalize constant GEPs to i8 type.
3213 if (!GEPEltType->isIntegerTy(8) && GEP.hasAllConstantIndices()) {
3214 APInt Offset(DL.getIndexTypeSizeInBits(GEPType), 0);
3215 if (GEP.accumulateConstantOffset(DL, Offset))
3216 return replaceInstUsesWith(
3217 GEP, Builder.CreatePtrAdd(PtrOp, Builder.getInt(Offset), "",
3218 GEP.getNoWrapFlags()));
3219 }
3220
3222 Value *Offset = EmitGEPOffset(cast<GEPOperator>(&GEP));
3223 Value *NewGEP =
3224 Builder.CreatePtrAdd(PtrOp, Offset, "", GEP.getNoWrapFlags());
3225 return replaceInstUsesWith(GEP, NewGEP);
3226 }
3227
3228 // Strip trailing zero indices.
3229 auto *LastIdx = dyn_cast<Constant>(Indices.back());
3230 if (LastIdx && LastIdx->isNullValue() && !LastIdx->getType()->isVectorTy()) {
3231 return replaceInstUsesWith(
3232 GEP, Builder.CreateGEP(GEP.getSourceElementType(), PtrOp,
3233 drop_end(Indices), "", GEP.getNoWrapFlags()));
3234 }
3235
3236 // Strip leading zero indices.
3237 auto *FirstIdx = dyn_cast<Constant>(Indices.front());
3238 if (FirstIdx && FirstIdx->isNullValue() &&
3239 !FirstIdx->getType()->isVectorTy()) {
3241 ++GTI;
3242 if (!GTI.isStruct())
3243 return replaceInstUsesWith(GEP, Builder.CreateGEP(GTI.getIndexedType(),
3244 GEP.getPointerOperand(),
3245 drop_begin(Indices), "",
3246 GEP.getNoWrapFlags()));
3247 }
3248
3249 // Scalarize vector operands; prefer splat-of-gep.as canonical form.
3250 // Note that this looses information about undef lanes; we run it after
3251 // demanded bits to partially mitigate that loss.
3252 if (GEPType->isVectorTy() && llvm::any_of(GEP.operands(), [](Value *Op) {
3253 return Op->getType()->isVectorTy() && getSplatValue(Op);
3254 })) {
3255 SmallVector<Value *> NewOps;
3256 for (auto &Op : GEP.operands()) {
3257 if (Op->getType()->isVectorTy())
3258 if (Value *Scalar = getSplatValue(Op)) {
3259 NewOps.push_back(Scalar);
3260 continue;
3261 }
3262 NewOps.push_back(Op);
3263 }
3264
3265 Value *Res = Builder.CreateGEP(GEP.getSourceElementType(), NewOps[0],
3266 ArrayRef(NewOps).drop_front(), GEP.getName(),
3267 GEP.getNoWrapFlags());
3268 if (!Res->getType()->isVectorTy()) {
3269 ElementCount EC = cast<VectorType>(GEPType)->getElementCount();
3270 Res = Builder.CreateVectorSplat(EC, Res);
3271 }
3272 return replaceInstUsesWith(GEP, Res);
3273 }
3274
3275 bool SeenNonZeroIndex = false;
3276 for (auto [IdxNum, Idx] : enumerate(Indices)) {
3277 auto *C = dyn_cast<Constant>(Idx);
3278 if (C && C->isNullValue())
3279 continue;
3280
3281 if (!SeenNonZeroIndex) {
3282 SeenNonZeroIndex = true;
3283 continue;
3284 }
3285
3286 // GEP has multiple non-zero indices: Split it.
3287 ArrayRef<Value *> FrontIndices = ArrayRef(Indices).take_front(IdxNum);
3288 Value *FrontGEP =
3289 Builder.CreateGEP(GEPEltType, PtrOp, FrontIndices,
3290 GEP.getName() + ".split", GEP.getNoWrapFlags());
3291
3292 SmallVector<Value *> BackIndices;
3293 BackIndices.push_back(Constant::getNullValue(NewScalarIndexTy));
3294 append_range(BackIndices, drop_begin(Indices, IdxNum));
3296 GetElementPtrInst::getIndexedType(GEPEltType, FrontIndices), FrontGEP,
3297 BackIndices, GEP.getNoWrapFlags());
3298 }
3299
3300 // Check to see if the inputs to the PHI node are getelementptr instructions.
3301 if (auto *PN = dyn_cast<PHINode>(PtrOp)) {
3302 if (Value *NewPtrOp = foldGEPOfPhi(GEP, PN, Builder))
3303 return replaceOperand(GEP, 0, NewPtrOp);
3304 }
3305
3306 if (auto *Src = dyn_cast<GEPOperator>(PtrOp))
3307 if (Instruction *I = visitGEPOfGEP(GEP, Src))
3308 return I;
3309
3310 if (GEP.getNumIndices() == 1) {
3311 unsigned AS = GEP.getPointerAddressSpace();
3312 if (GEP.getOperand(1)->getType()->getScalarSizeInBits() ==
3313 DL.getIndexSizeInBits(AS)) {
3314 uint64_t TyAllocSize = DL.getTypeAllocSize(GEPEltType).getFixedValue();
3315
3316 if (TyAllocSize == 1) {
3317 // Canonicalize (gep i8* X, (ptrtoint Y)-(ptrtoint X)) to (bitcast Y),
3318 // but only if the result pointer is only used as if it were an integer,
3319 // or both point to the same underlying object (otherwise provenance is
3320 // not necessarily retained).
3321 Value *X = GEP.getPointerOperand();
3322 Value *Y;
3323 if (match(GEP.getOperand(1),
3325 GEPType == Y->getType()) {
3326 bool HasSameUnderlyingObject =
3328 bool Changed = false;
3329 GEP.replaceUsesWithIf(Y, [&](Use &U) {
3330 bool ShouldReplace = HasSameUnderlyingObject ||
3331 isa<ICmpInst>(U.getUser()) ||
3332 isa<PtrToIntInst>(U.getUser());
3333 Changed |= ShouldReplace;
3334 return ShouldReplace;
3335 });
3336 return Changed ? &GEP : nullptr;
3337 }
3338 } else if (auto *ExactIns =
3339 dyn_cast<PossiblyExactOperator>(GEP.getOperand(1))) {
3340 // Canonicalize (gep T* X, V / sizeof(T)) to (gep i8* X, V)
3341 Value *V;
3342 if (ExactIns->isExact()) {
3343 if ((has_single_bit(TyAllocSize) &&
3344 match(GEP.getOperand(1),
3345 m_Shr(m_Value(V),
3346 m_SpecificInt(countr_zero(TyAllocSize))))) ||
3347 match(GEP.getOperand(1),
3348 m_IDiv(m_Value(V), m_SpecificInt(TyAllocSize)))) {
3349 return GetElementPtrInst::Create(Builder.getInt8Ty(),
3350 GEP.getPointerOperand(), V,
3351 GEP.getNoWrapFlags());
3352 }
3353 }
3354 if (ExactIns->isExact() && ExactIns->hasOneUse()) {
3355 // Try to canonicalize non-i8 element type to i8 if the index is an
3356 // exact instruction. If the index is an exact instruction (div/shr)
3357 // with a constant RHS, we can fold the non-i8 element scale into the
3358 // div/shr (similiar to the mul case, just inverted).
3359 const APInt *C;
3360 std::optional<APInt> NewC;
3361 if (has_single_bit(TyAllocSize) &&
3362 match(ExactIns, m_Shr(m_Value(V), m_APInt(C))) &&
3363 C->uge(countr_zero(TyAllocSize)))
3364 NewC = *C - countr_zero(TyAllocSize);
3365 else if (match(ExactIns, m_UDiv(m_Value(V), m_APInt(C)))) {
3366 APInt Quot;
3367 uint64_t Rem;
3368 APInt::udivrem(*C, TyAllocSize, Quot, Rem);
3369 if (Rem == 0)
3370 NewC = Quot;
3371 } else if (match(ExactIns, m_SDiv(m_Value(V), m_APInt(C)))) {
3372 APInt Quot;
3373 int64_t Rem;
3374 APInt::sdivrem(*C, TyAllocSize, Quot, Rem);
3375 // For sdiv we need to make sure we arent creating INT_MIN / -1.
3376 if (!Quot.isAllOnes() && Rem == 0)
3377 NewC = Quot;
3378 }
3379
3380 if (NewC.has_value()) {
3381 Value *NewOp = Builder.CreateBinOp(
3382 static_cast<Instruction::BinaryOps>(ExactIns->getOpcode()), V,
3383 ConstantInt::get(V->getType(), *NewC));
3384 cast<BinaryOperator>(NewOp)->setIsExact();
3385 return GetElementPtrInst::Create(Builder.getInt8Ty(),
3386 GEP.getPointerOperand(), NewOp,
3387 GEP.getNoWrapFlags());
3388 }
3389 }
3390 }
3391 }
3392 }
3393 // We do not handle pointer-vector geps here.
3394 if (GEPType->isVectorTy())
3395 return nullptr;
3396
3397 if (!GEP.isInBounds()) {
3398 unsigned IdxWidth =
3399 DL.getIndexSizeInBits(PtrOp->getType()->getPointerAddressSpace());
3400 APInt BasePtrOffset(IdxWidth, 0);
3401 Value *UnderlyingPtrOp =
3402 PtrOp->stripAndAccumulateInBoundsConstantOffsets(DL, BasePtrOffset);
3403 bool CanBeNull, CanBeFreed;
3404 uint64_t DerefBytes = UnderlyingPtrOp->getPointerDereferenceableBytes(
3405 DL, CanBeNull, CanBeFreed);
3406 if (!CanBeNull && !CanBeFreed && DerefBytes != 0) {
3407 if (GEP.accumulateConstantOffset(DL, BasePtrOffset) &&
3408 BasePtrOffset.isNonNegative()) {
3409 APInt AllocSize(IdxWidth, DerefBytes);
3410 if (BasePtrOffset.ule(AllocSize)) {
3412 GEP.getSourceElementType(), PtrOp, Indices, GEP.getName());
3413 }
3414 }
3415 }
3416 }
3417
3418 // nusw + nneg -> nuw
3419 if (GEP.hasNoUnsignedSignedWrap() && !GEP.hasNoUnsignedWrap() &&
3420 all_of(GEP.indices(), [&](Value *Idx) {
3421 return isKnownNonNegative(Idx, SQ.getWithInstruction(&GEP));
3422 })) {
3423 GEP.setNoWrapFlags(GEP.getNoWrapFlags() | GEPNoWrapFlags::noUnsignedWrap());
3424 return &GEP;
3425 }
3426
3427 // These rewrites are trying to preserve inbounds/nuw attributes. So we want
3428 // to do this after having tried to derive "nuw" above.
3429 if (GEP.getNumIndices() == 1) {
3430 // Given (gep p, x+y) we want to determine the common nowrap flags for both
3431 // geps if transforming into (gep (gep p, x), y).
3432 auto GetPreservedNoWrapFlags = [&](bool AddIsNUW) {
3433 // We can preserve both "inbounds nuw", "nusw nuw" and "nuw" if we know
3434 // that x + y does not have unsigned wrap.
3435 if (GEP.hasNoUnsignedWrap() && AddIsNUW)
3436 return GEP.getNoWrapFlags();
3437 return GEPNoWrapFlags::none();
3438 };
3439
3440 // Try to replace ADD + GEP with GEP + GEP.
3441 Value *Idx1, *Idx2;
3442 if (match(GEP.getOperand(1),
3443 m_OneUse(m_AddLike(m_Value(Idx1), m_Value(Idx2))))) {
3444 // %idx = add i64 %idx1, %idx2
3445 // %gep = getelementptr i32, ptr %ptr, i64 %idx
3446 // as:
3447 // %newptr = getelementptr i32, ptr %ptr, i64 %idx1
3448 // %newgep = getelementptr i32, ptr %newptr, i64 %idx2
3449 bool NUW = match(GEP.getOperand(1), m_NUWAddLike(m_Value(), m_Value()));
3450 GEPNoWrapFlags NWFlags = GetPreservedNoWrapFlags(NUW);
3451 auto *NewPtr =
3452 Builder.CreateGEP(GEP.getSourceElementType(), GEP.getPointerOperand(),
3453 Idx1, "", NWFlags);
3454 return replaceInstUsesWith(GEP,
3455 Builder.CreateGEP(GEP.getSourceElementType(),
3456 NewPtr, Idx2, "", NWFlags));
3457 }
3458 ConstantInt *C;
3459 if (match(GEP.getOperand(1), m_OneUse(m_SExtLike(m_OneUse(m_NSWAddLike(
3460 m_Value(Idx1), m_ConstantInt(C))))))) {
3461 // %add = add nsw i32 %idx1, idx2
3462 // %sidx = sext i32 %add to i64
3463 // %gep = getelementptr i32, ptr %ptr, i64 %sidx
3464 // as:
3465 // %newptr = getelementptr i32, ptr %ptr, i32 %idx1
3466 // %newgep = getelementptr i32, ptr %newptr, i32 idx2
3467 bool NUW = match(GEP.getOperand(1),
3469 GEPNoWrapFlags NWFlags = GetPreservedNoWrapFlags(NUW);
3470 auto *NewPtr = Builder.CreateGEP(
3471 GEP.getSourceElementType(), GEP.getPointerOperand(),
3472 Builder.CreateSExt(Idx1, GEP.getOperand(1)->getType()), "", NWFlags);
3473 return replaceInstUsesWith(
3474 GEP,
3475 Builder.CreateGEP(GEP.getSourceElementType(), NewPtr,
3476 Builder.CreateSExt(C, GEP.getOperand(1)->getType()),
3477 "", NWFlags));
3478 }
3479 }
3480
3482 return R;
3483
3484 return nullptr;
3485}
3486
3488 Instruction *AI) {
3490 return true;
3491 if (auto *LI = dyn_cast<LoadInst>(V))
3492 return isa<GlobalVariable>(LI->getPointerOperand());
3493 // Two distinct allocations will never be equal.
3494 return isAllocLikeFn(V, &TLI) && V != AI;
3495}
3496
3497/// Given a call CB which uses an address UsedV, return true if we can prove the
3498/// call's only possible effect is storing to V.
3499static bool isRemovableWrite(CallBase &CB, Value *UsedV,
3500 const TargetLibraryInfo &TLI) {
3501 if (!CB.use_empty())
3502 // TODO: add recursion if returned attribute is present
3503 return false;
3504
3505 if (CB.isTerminator())
3506 // TODO: remove implementation restriction
3507 return false;
3508
3509 if (!CB.willReturn() || !CB.doesNotThrow())
3510 return false;
3511
3512 // If the only possible side effect of the call is writing to the alloca,
3513 // and the result isn't used, we can safely remove any reads implied by the
3514 // call including those which might read the alloca itself.
3515 std::optional<MemoryLocation> Dest = MemoryLocation::getForDest(&CB, TLI);
3516 return Dest && Dest->Ptr == UsedV;
3517}
3518
3519static std::optional<ModRefInfo>
3521 const TargetLibraryInfo &TLI, bool KnowInit) {
3523 const std::optional<StringRef> Family = getAllocationFamily(AI, &TLI);
3524 Worklist.push_back(AI);
3526
3527 do {
3528 Instruction *PI = Worklist.pop_back_val();
3529 for (User *U : PI->users()) {
3531 switch (I->getOpcode()) {
3532 default:
3533 // Give up the moment we see something we can't handle.
3534 return std::nullopt;
3535
3536 case Instruction::AddrSpaceCast:
3537 case Instruction::BitCast:
3538 case Instruction::GetElementPtr:
3539 Users.emplace_back(I);
3540 Worklist.push_back(I);
3541 continue;
3542
3543 case Instruction::ICmp: {
3544 ICmpInst *ICI = cast<ICmpInst>(I);
3545 // We can fold eq/ne comparisons with null to false/true, respectively.
3546 // We also fold comparisons in some conditions provided the alloc has
3547 // not escaped (see isNeverEqualToUnescapedAlloc).
3548 if (!ICI->isEquality())
3549 return std::nullopt;
3550 unsigned OtherIndex = (ICI->getOperand(0) == PI) ? 1 : 0;
3551 if (!isNeverEqualToUnescapedAlloc(ICI->getOperand(OtherIndex), TLI, AI))
3552 return std::nullopt;
3553
3554 // Do not fold compares to aligned_alloc calls, as they may have to
3555 // return null in case the required alignment cannot be satisfied,
3556 // unless we can prove that both alignment and size are valid.
3557 auto AlignmentAndSizeKnownValid = [](CallBase *CB) {
3558 // Check if alignment and size of a call to aligned_alloc is valid,
3559 // that is alignment is a power-of-2 and the size is a multiple of the
3560 // alignment.
3561 const APInt *Alignment;
3562 const APInt *Size;
3563 return match(CB->getArgOperand(0), m_APInt(Alignment)) &&
3564 match(CB->getArgOperand(1), m_APInt(Size)) &&
3565 Alignment->isPowerOf2() && Size->urem(*Alignment).isZero();
3566 };
3567 auto *CB = dyn_cast<CallBase>(AI);
3568 LibFunc TheLibFunc;
3569 if (CB && TLI.getLibFunc(*CB->getCalledFunction(), TheLibFunc) &&
3570 TLI.has(TheLibFunc) && TheLibFunc == LibFunc_aligned_alloc &&
3571 !AlignmentAndSizeKnownValid(CB))
3572 return std::nullopt;
3573 Users.emplace_back(I);
3574 continue;
3575 }
3576
3577 case Instruction::Call:
3578 // Ignore no-op and store intrinsics.
3580 switch (II->getIntrinsicID()) {
3581 default:
3582 return std::nullopt;
3583
3584 case Intrinsic::memmove:
3585 case Intrinsic::memcpy:
3586 case Intrinsic::memset: {
3588 if (MI->isVolatile())
3589 return std::nullopt;
3590 // Note: this could also be ModRef, but we can still interpret that
3591 // as just Mod in that case.
3592 ModRefInfo NewAccess =
3593 MI->getRawDest() == PI ? ModRefInfo::Mod : ModRefInfo::Ref;
3594 if ((Access & ~NewAccess) != ModRefInfo::NoModRef)
3595 return std::nullopt;
3596 Access |= NewAccess;
3597 [[fallthrough]];
3598 }
3599 case Intrinsic::assume:
3600 case Intrinsic::invariant_start:
3601 case Intrinsic::invariant_end:
3602 case Intrinsic::lifetime_start:
3603 case Intrinsic::lifetime_end:
3604 case Intrinsic::objectsize:
3605 Users.emplace_back(I);
3606 continue;
3607 case Intrinsic::launder_invariant_group:
3608 case Intrinsic::strip_invariant_group:
3609 Users.emplace_back(I);
3610 Worklist.push_back(I);
3611 continue;
3612 }
3613 }
3614
3615 if (Family && getFreedOperand(cast<CallBase>(I), &TLI) == PI &&
3616 getAllocationFamily(I, &TLI) == Family) {
3617 Users.emplace_back(I);
3618 continue;
3619 }
3620
3621 if (Family && getReallocatedOperand(cast<CallBase>(I)) == PI &&
3622 getAllocationFamily(I, &TLI) == Family) {
3623 Users.emplace_back(I);
3624 Worklist.push_back(I);
3625 continue;
3626 }
3627
3628 if (!isRefSet(Access) &&
3629 isRemovableWrite(*cast<CallBase>(I), PI, TLI)) {
3631 Users.emplace_back(I);
3632 continue;
3633 }
3634
3635 return std::nullopt;
3636
3637 case Instruction::Store: {
3639 if (SI->isVolatile() || SI->getPointerOperand() != PI)
3640 return std::nullopt;
3641 if (isRefSet(Access))
3642 return std::nullopt;
3644 Users.emplace_back(I);
3645 continue;
3646 }
3647
3648 case Instruction::Load: {
3649 LoadInst *LI = cast<LoadInst>(I);
3650 if (LI->isVolatile() || LI->getPointerOperand() != PI)
3651 return std::nullopt;
3652 if (isModSet(Access))
3653 return std::nullopt;
3655 Users.emplace_back(I);
3656 continue;
3657 }
3658 }
3659 llvm_unreachable("missing a return?");
3660 }
3661 } while (!Worklist.empty());
3662
3664 return Access;
3665}
3666
3669
3670 // If we have a malloc call which is only used in any amount of comparisons to
3671 // null and free calls, delete the calls and replace the comparisons with true
3672 // or false as appropriate.
3673
3674 // This is based on the principle that we can substitute our own allocation
3675 // function (which will never return null) rather than knowledge of the
3676 // specific function being called. In some sense this can change the permitted
3677 // outputs of a program (when we convert a malloc to an alloca, the fact that
3678 // the allocation is now on the stack is potentially visible, for example),
3679 // but we believe in a permissible manner.
3681
3682 // If we are removing an alloca with a dbg.declare, insert dbg.value calls
3683 // before each store.
3685 std::unique_ptr<DIBuilder> DIB;
3686 if (isa<AllocaInst>(MI)) {
3687 findDbgUsers(&MI, DVRs);
3688 DIB.reset(new DIBuilder(*MI.getModule(), /*AllowUnresolved=*/false));
3689 }
3690
3691 // Determine what getInitialValueOfAllocation would return without actually
3692 // allocating the result.
3693 bool KnowInitUndef = false;
3694 bool KnowInitZero = false;
3695 Constant *Init =
3697 if (Init) {
3698 if (isa<UndefValue>(Init))
3699 KnowInitUndef = true;
3700 else if (Init->isNullValue())
3701 KnowInitZero = true;
3702 }
3703 // The various sanitizers don't actually return undef memory, but rather
3704 // memory initialized with special forms of runtime poison
3705 auto &F = *MI.getFunction();
3706 if (F.hasFnAttribute(Attribute::SanitizeMemory) ||
3707 F.hasFnAttribute(Attribute::SanitizeAddress))
3708 KnowInitUndef = false;
3709
3710 auto Removable =
3711 isAllocSiteRemovable(&MI, Users, TLI, KnowInitZero | KnowInitUndef);
3712 if (Removable) {
3713 for (WeakTrackingVH &User : Users) {
3714 // Lowering all @llvm.objectsize and MTI calls first because they may use
3715 // a bitcast/GEP of the alloca we are removing.
3716 if (!User)
3717 continue;
3718
3720
3722 if (II->getIntrinsicID() == Intrinsic::objectsize) {
3723 SmallVector<Instruction *> InsertedInstructions;
3724 Value *Result = lowerObjectSizeCall(
3725 II, DL, &TLI, AA, /*MustSucceed=*/true, &InsertedInstructions);
3726 for (Instruction *Inserted : InsertedInstructions)
3727 Worklist.add(Inserted);
3728 replaceInstUsesWith(*I, Result);
3730 User = nullptr; // Skip examining in the next loop.
3731 continue;
3732 }
3733 if (auto *MTI = dyn_cast<MemTransferInst>(I)) {
3734 if (KnowInitZero && isRefSet(*Removable)) {
3736 Builder.SetInsertPoint(MTI);
3737 auto *M = Builder.CreateMemSet(
3738 MTI->getRawDest(),
3739 ConstantInt::get(Type::getInt8Ty(MI.getContext()), 0),
3740 MTI->getLength(), MTI->getDestAlign());
3741 M->copyMetadata(*MTI);
3742 }
3743 }
3744 }
3745 }
3746 for (WeakTrackingVH &User : Users) {
3747 if (!User)
3748 continue;
3749
3751
3752 if (ICmpInst *C = dyn_cast<ICmpInst>(I)) {
3754 ConstantInt::get(Type::getInt1Ty(C->getContext()),
3755 C->isFalseWhenEqual()));
3756 } else if (auto *SI = dyn_cast<StoreInst>(I)) {
3757 for (auto *DVR : DVRs)
3758 if (DVR->isAddressOfVariable())
3760 } else {
3761 // Casts, GEP, or anything else: we're about to delete this instruction,
3762 // so it can not have any valid uses.
3763 Constant *Replace;
3764 if (isa<LoadInst>(I)) {
3765 assert(KnowInitZero || KnowInitUndef);
3766 Replace = KnowInitUndef ? UndefValue::get(I->getType())
3767 : Constant::getNullValue(I->getType());
3768 } else
3769 Replace = PoisonValue::get(I->getType());
3770 replaceInstUsesWith(*I, Replace);
3771 }
3773 }
3774
3776 // Replace invoke with a NOP intrinsic to maintain the original CFG
3777 Module *M = II->getModule();
3778 Function *F = Intrinsic::getOrInsertDeclaration(M, Intrinsic::donothing);
3779 auto *NewII = InvokeInst::Create(
3780 F, II->getNormalDest(), II->getUnwindDest(), {}, "", II->getParent());
3781 NewII->setDebugLoc(II->getDebugLoc());
3782 }
3783
3784 // Remove debug intrinsics which describe the value contained within the
3785 // alloca. In addition to removing dbg.{declare,addr} which simply point to
3786 // the alloca, remove dbg.value(<alloca>, ..., DW_OP_deref)'s as well, e.g.:
3787 //
3788 // ```
3789 // define void @foo(i32 %0) {
3790 // %a = alloca i32 ; Deleted.
3791 // store i32 %0, i32* %a
3792 // dbg.value(i32 %0, "arg0") ; Not deleted.
3793 // dbg.value(i32* %a, "arg0", DW_OP_deref) ; Deleted.
3794 // call void @trivially_inlinable_no_op(i32* %a)
3795 // ret void
3796 // }
3797 // ```
3798 //
3799 // This may not be required if we stop describing the contents of allocas
3800 // using dbg.value(<alloca>, ..., DW_OP_deref), but we currently do this in
3801 // the LowerDbgDeclare utility.
3802 //
3803 // If there is a dead store to `%a` in @trivially_inlinable_no_op, the
3804 // "arg0" dbg.value may be stale after the call. However, failing to remove
3805 // the DW_OP_deref dbg.value causes large gaps in location coverage.
3806 //
3807 // FIXME: the Assignment Tracking project has now likely made this
3808 // redundant (and it's sometimes harmful).
3809 for (auto *DVR : DVRs)
3810 if (DVR->isAddressOfVariable() || DVR->getExpression()->startsWithDeref())
3811 DVR->eraseFromParent();
3812
3813 return eraseInstFromFunction(MI);
3814 }
3815 return nullptr;
3816}
3817
3818/// Move the call to free before a NULL test.
3819///
3820/// Check if this free is accessed after its argument has been test
3821/// against NULL (property 0).
3822/// If yes, it is legal to move this call in its predecessor block.
3823///
3824/// The move is performed only if the block containing the call to free
3825/// will be removed, i.e.:
3826/// 1. it has only one predecessor P, and P has two successors
3827/// 2. it contains the call, noops, and an unconditional branch
3828/// 3. its successor is the same as its predecessor's successor
3829///
3830/// The profitability is out-of concern here and this function should
3831/// be called only if the caller knows this transformation would be
3832/// profitable (e.g., for code size).
3834 const DataLayout &DL) {
3835 Value *Op = FI.getArgOperand(0);
3836 BasicBlock *FreeInstrBB = FI.getParent();
3837 BasicBlock *PredBB = FreeInstrBB->getSinglePredecessor();
3838
3839 // Validate part of constraint #1: Only one predecessor
3840 // FIXME: We can extend the number of predecessor, but in that case, we
3841 // would duplicate the call to free in each predecessor and it may
3842 // not be profitable even for code size.
3843 if (!PredBB)
3844 return nullptr;
3845
3846 // Validate constraint #2: Does this block contains only the call to
3847 // free, noops, and an unconditional branch?
3848 BasicBlock *SuccBB;
3849 Instruction *FreeInstrBBTerminator = FreeInstrBB->getTerminator();
3850 if (!match(FreeInstrBBTerminator, m_UnconditionalBr(SuccBB)))
3851 return nullptr;
3852
3853 // If there are only 2 instructions in the block, at this point,
3854 // this is the call to free and unconditional.
3855 // If there are more than 2 instructions, check that they are noops
3856 // i.e., they won't hurt the performance of the generated code.
3857 if (FreeInstrBB->size() != 2) {
3858 for (const Instruction &Inst : FreeInstrBB->instructionsWithoutDebug()) {
3859 if (&Inst == &FI || &Inst == FreeInstrBBTerminator)
3860 continue;
3861 auto *Cast = dyn_cast<CastInst>(&Inst);
3862 if (!Cast || !Cast->isNoopCast(DL))
3863 return nullptr;
3864 }
3865 }
3866 // Validate the rest of constraint #1 by matching on the pred branch.
3867 Instruction *TI = PredBB->getTerminator();
3868 BasicBlock *TrueBB, *FalseBB;
3869 CmpPredicate Pred;
3870 if (!match(TI, m_Br(m_ICmp(Pred,
3872 m_Specific(Op->stripPointerCasts())),
3873 m_Zero()),
3874 TrueBB, FalseBB)))
3875 return nullptr;
3876 if (Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
3877 return nullptr;
3878
3879 // Validate constraint #3: Ensure the null case just falls through.
3880 if (SuccBB != (Pred == ICmpInst::ICMP_EQ ? TrueBB : FalseBB))
3881 return nullptr;
3882 assert(FreeInstrBB == (Pred == ICmpInst::ICMP_EQ ? FalseBB : TrueBB) &&
3883 "Broken CFG: missing edge from predecessor to successor");
3884
3885 // At this point, we know that everything in FreeInstrBB can be moved
3886 // before TI.
3887 for (Instruction &Instr : llvm::make_early_inc_range(*FreeInstrBB)) {
3888 if (&Instr == FreeInstrBBTerminator)
3889 break;
3890 Instr.moveBeforePreserving(TI->getIterator());
3891 }
3892 assert(FreeInstrBB->size() == 1 &&
3893 "Only the branch instruction should remain");
3894
3895 // Now that we've moved the call to free before the NULL check, we have to
3896 // remove any attributes on its parameter that imply it's non-null, because
3897 // those attributes might have only been valid because of the NULL check, and
3898 // we can get miscompiles if we keep them. This is conservative if non-null is
3899 // also implied by something other than the NULL check, but it's guaranteed to
3900 // be correct, and the conservativeness won't matter in practice, since the
3901 // attributes are irrelevant for the call to free itself and the pointer
3902 // shouldn't be used after the call.
3903 AttributeList Attrs = FI.getAttributes();
3904 Attrs = Attrs.removeParamAttribute(FI.getContext(), 0, Attribute::NonNull);
3905 Attribute Dereferenceable = Attrs.getParamAttr(0, Attribute::Dereferenceable);
3906 if (Dereferenceable.isValid()) {
3907 uint64_t Bytes = Dereferenceable.getDereferenceableBytes();
3908 Attrs = Attrs.removeParamAttribute(FI.getContext(), 0,
3909 Attribute::Dereferenceable);
3910 Attrs = Attrs.addDereferenceableOrNullParamAttr(FI.getContext(), 0, Bytes);
3911 }
3912 FI.setAttributes(Attrs);
3913
3914 return &FI;
3915}
3916
3918 // free undef -> unreachable.
3919 if (isa<UndefValue>(Op)) {
3920 // Leave a marker since we can't modify the CFG here.
3922 return eraseInstFromFunction(FI);
3923 }
3924
3925 // If we have 'free null' delete the instruction. This can happen in stl code
3926 // when lots of inlining happens.
3928 return eraseInstFromFunction(FI);
3929
3930 // If we had free(realloc(...)) with no intervening uses, then eliminate the
3931 // realloc() entirely.
3933 if (CI && CI->hasOneUse())
3934 if (Value *ReallocatedOp = getReallocatedOperand(CI))
3935 return eraseInstFromFunction(*replaceInstUsesWith(*CI, ReallocatedOp));
3936
3937 // If we optimize for code size, try to move the call to free before the null
3938 // test so that simplify cfg can remove the empty block and dead code
3939 // elimination the branch. I.e., helps to turn something like:
3940 // if (foo) free(foo);
3941 // into
3942 // free(foo);
3943 //
3944 // Note that we can only do this for 'free' and not for any flavor of
3945 // 'operator delete'; there is no 'operator delete' symbol for which we are
3946 // permitted to invent a call, even if we're passing in a null pointer.
3947 if (MinimizeSize) {
3948 LibFunc Func;
3949 if (TLI.getLibFunc(FI, Func) && TLI.has(Func) && Func == LibFunc_free)
3951 return I;
3952 }
3953
3954 return nullptr;
3955}
3956
3958 Value *RetVal = RI.getReturnValue();
3959 if (!RetVal)
3960 return nullptr;
3961
3962 Function *F = RI.getFunction();
3963 Type *RetTy = RetVal->getType();
3964 if (RetTy->isPointerTy()) {
3965 bool HasDereferenceable =
3966 F->getAttributes().getRetDereferenceableBytes() > 0;
3967 if (F->hasRetAttribute(Attribute::NonNull) ||
3968 (HasDereferenceable &&
3970 if (Value *V = simplifyNonNullOperand(RetVal, HasDereferenceable))
3971 return replaceOperand(RI, 0, V);
3972 }
3973 }
3974
3975 if (!AttributeFuncs::isNoFPClassCompatibleType(RetTy))
3976 return nullptr;
3977
3978 FPClassTest ReturnClass = F->getAttributes().getRetNoFPClass();
3979 if (ReturnClass == fcNone)
3980 return nullptr;
3981
3982 KnownFPClass KnownClass;
3983 Value *Simplified =
3984 SimplifyDemandedUseFPClass(RetVal, ~ReturnClass, KnownClass, &RI);
3985 if (!Simplified)
3986 return nullptr;
3987
3988 return ReturnInst::Create(RI.getContext(), Simplified);
3989}
3990
3991// WARNING: keep in sync with SimplifyCFGOpt::simplifyUnreachable()!
3993 // Try to remove the previous instruction if it must lead to unreachable.
3994 // This includes instructions like stores and "llvm.assume" that may not get
3995 // removed by simple dead code elimination.
3996 bool Changed = false;
3997 while (Instruction *Prev = I.getPrevNode()) {
3998 // While we theoretically can erase EH, that would result in a block that
3999 // used to start with an EH no longer starting with EH, which is invalid.
4000 // To make it valid, we'd need to fixup predecessors to no longer refer to
4001 // this block, but that changes CFG, which is not allowed in InstCombine.
4002 if (Prev->isEHPad())
4003 break; // Can not drop any more instructions. We're done here.
4004
4006 break; // Can not drop any more instructions. We're done here.
4007 // Otherwise, this instruction can be freely erased,
4008 // even if it is not side-effect free.
4009
4010 // A value may still have uses before we process it here (for example, in
4011 // another unreachable block), so convert those to poison.
4012 replaceInstUsesWith(*Prev, PoisonValue::get(Prev->getType()));
4013 eraseInstFromFunction(*Prev);
4014 Changed = true;
4015 }
4016 return Changed;
4017}
4018
4023
4025 assert(BI.isUnconditional() && "Only for unconditional branches.");
4026
4027 // If this store is the second-to-last instruction in the basic block
4028 // (excluding debug info) and if the block ends with
4029 // an unconditional branch, try to move the store to the successor block.
4030
4031 auto GetLastSinkableStore = [](BasicBlock::iterator BBI) {
4032 BasicBlock::iterator FirstInstr = BBI->getParent()->begin();
4033 do {
4034 if (BBI != FirstInstr)
4035 --BBI;
4036 } while (BBI != FirstInstr && BBI->isDebugOrPseudoInst());
4037
4038 return dyn_cast<StoreInst>(BBI);
4039 };
4040
4041 if (StoreInst *SI = GetLastSinkableStore(BasicBlock::iterator(BI)))
4043 return &BI;
4044
4045 return nullptr;
4046}
4047
4050 if (!DeadEdges.insert({From, To}).second)
4051 return;
4052
4053 // Replace phi node operands in successor with poison.
4054 for (PHINode &PN : To->phis())
4055 for (Use &U : PN.incoming_values())
4056 if (PN.getIncomingBlock(U) == From && !isa<PoisonValue>(U)) {
4057 replaceUse(U, PoisonValue::get(PN.getType()));
4058 addToWorklist(&PN);
4059 MadeIRChange = true;
4060 }
4061
4062 Worklist.push_back(To);
4063}
4064
4065// Under the assumption that I is unreachable, remove it and following
4066// instructions. Changes are reported directly to MadeIRChange.
4069 BasicBlock *BB = I->getParent();
4070 for (Instruction &Inst : make_early_inc_range(
4071 make_range(std::next(BB->getTerminator()->getReverseIterator()),
4072 std::next(I->getReverseIterator())))) {
4073 if (!Inst.use_empty() && !Inst.getType()->isTokenTy()) {
4074 replaceInstUsesWith(Inst, PoisonValue::get(Inst.getType()));
4075 MadeIRChange = true;
4076 }
4077 if (Inst.isEHPad() || Inst.getType()->isTokenTy())
4078 continue;
4079 // RemoveDIs: erase debug-info on this instruction manually.
4080 Inst.dropDbgRecords();
4082 MadeIRChange = true;
4083 }
4084
4087 MadeIRChange = true;
4088 for (Value *V : Changed)
4090 }
4091
4092 // Handle potentially dead successors.
4093 for (BasicBlock *Succ : successors(BB))
4094 addDeadEdge(BB, Succ, Worklist);
4095}
4096
4099 while (!Worklist.empty()) {
4100 BasicBlock *BB = Worklist.pop_back_val();
4101 if (!all_of(predecessors(BB), [&](BasicBlock *Pred) {
4102 return DeadEdges.contains({Pred, BB}) || DT.dominates(BB, Pred);
4103 }))
4104 continue;
4105
4107 }
4108}
4109
4111 BasicBlock *LiveSucc) {
4113 for (BasicBlock *Succ : successors(BB)) {
4114 // The live successor isn't dead.
4115 if (Succ == LiveSucc)
4116 continue;
4117
4118 addDeadEdge(BB, Succ, Worklist);
4119 }
4120
4122}
4123
4125 if (BI.isUnconditional())
4127
4128 // Change br (not X), label True, label False to: br X, label False, True
4129 Value *Cond = BI.getCondition();
4130 Value *X;
4131 if (match(Cond, m_Not(m_Value(X))) && !isa<Constant>(X)) {
4132 // Swap Destinations and condition...
4133 BI.swapSuccessors();
4134 if (BPI)
4135 BPI->swapSuccEdgesProbabilities(BI.getParent());
4136 return replaceOperand(BI, 0, X);
4137 }
4138
4139 // Canonicalize logical-and-with-invert as logical-or-with-invert.
4140 // This is done by inverting the condition and swapping successors:
4141 // br (X && !Y), T, F --> br !(X && !Y), F, T --> br (!X || Y), F, T
4142 Value *Y;
4143 if (isa<SelectInst>(Cond) &&
4144 match(Cond,
4146 Value *NotX = Builder.CreateNot(X, "not." + X->getName());
4147 Value *Or = Builder.CreateLogicalOr(NotX, Y);
4148 BI.swapSuccessors();
4149 if (BPI)
4150 BPI->swapSuccEdgesProbabilities(BI.getParent());
4151 return replaceOperand(BI, 0, Or);
4152 }
4153
4154 // If the condition is irrelevant, remove the use so that other
4155 // transforms on the condition become more effective.
4156 if (!isa<ConstantInt>(Cond) && BI.getSuccessor(0) == BI.getSuccessor(1))
4157 return replaceOperand(BI, 0, ConstantInt::getFalse(Cond->getType()));
4158
4159 // Canonicalize, for example, fcmp_one -> fcmp_oeq.
4160 CmpPredicate Pred;
4161 if (match(Cond, m_OneUse(m_FCmp(Pred, m_Value(), m_Value()))) &&
4162 !isCanonicalPredicate(Pred)) {
4163 // Swap destinations and condition.
4164 auto *Cmp = cast<CmpInst>(Cond);
4165 Cmp->setPredicate(CmpInst::getInversePredicate(Pred));
4166 BI.swapSuccessors();
4167 if (BPI)
4168 BPI->swapSuccEdgesProbabilities(BI.getParent());
4169 Worklist.push(Cmp);
4170 return &BI;
4171 }
4172
4173 if (isa<UndefValue>(Cond)) {
4174 handlePotentiallyDeadSuccessors(BI.getParent(), /*LiveSucc*/ nullptr);
4175 return nullptr;
4176 }
4177 if (auto *CI = dyn_cast<ConstantInt>(Cond)) {
4179 BI.getSuccessor(!CI->getZExtValue()));
4180 return nullptr;
4181 }
4182
4183 // Replace all dominated uses of the condition with true/false
4184 // Ignore constant expressions to avoid iterating over uses on other
4185 // functions.
4186 if (!isa<Constant>(Cond) && BI.getSuccessor(0) != BI.getSuccessor(1)) {
4187 for (auto &U : make_early_inc_range(Cond->uses())) {
4188 BasicBlockEdge Edge0(BI.getParent(), BI.getSuccessor(0));
4189 if (DT.dominates(Edge0, U)) {
4190 replaceUse(U, ConstantInt::getTrue(Cond->getType()));
4191 addToWorklist(cast<Instruction>(U.getUser()));
4192 continue;
4193 }
4194 BasicBlockEdge Edge1(BI.getParent(), BI.getSuccessor(1));
4195 if (DT.dominates(Edge1, U)) {
4196 replaceUse(U, ConstantInt::getFalse(Cond->getType()));
4197 addToWorklist(cast<Instruction>(U.getUser()));
4198 }
4199 }
4200 }
4201
4202 DC.registerBranch(&BI);
4203 return nullptr;
4204}
4205
4206// Replaces (switch (select cond, X, C)/(select cond, C, X)) with (switch X) if
4207// we can prove that both (switch C) and (switch X) go to the default when cond
4208// is false/true.
4211 bool IsTrueArm) {
4212 unsigned CstOpIdx = IsTrueArm ? 1 : 2;
4213 auto *C = dyn_cast<ConstantInt>(Select->getOperand(CstOpIdx));
4214 if (!C)
4215 return nullptr;
4216
4217 BasicBlock *CstBB = SI.findCaseValue(C)->getCaseSuccessor();
4218 if (CstBB != SI.getDefaultDest())
4219 return nullptr;
4220 Value *X = Select->getOperand(3 - CstOpIdx);
4221 CmpPredicate Pred;
4222 const APInt *RHSC;
4223 if (!match(Select->getCondition(),
4224 m_ICmp(Pred, m_Specific(X), m_APInt(RHSC))))
4225 return nullptr;
4226 if (IsTrueArm)
4227 Pred = ICmpInst::getInversePredicate(Pred);
4228
4229 // See whether we can replace the select with X
4231 for (auto Case : SI.cases())
4232 if (!CR.contains(Case.getCaseValue()->getValue()))
4233 return nullptr;
4234
4235 return X;
4236}
4237
4239 Value *Cond = SI.getCondition();
4240 Value *Op0;
4241 ConstantInt *AddRHS;
4242 if (match(Cond, m_Add(m_Value(Op0), m_ConstantInt(AddRHS)))) {
4243 // Change 'switch (X+4) case 1:' into 'switch (X) case -3'.
4244 for (auto Case : SI.cases()) {
4245 Constant *NewCase = ConstantExpr::getSub(Case.getCaseValue(), AddRHS);
4246 assert(isa<ConstantInt>(NewCase) &&
4247 "Result of expression should be constant");
4248 Case.setValue(cast<ConstantInt>(NewCase));
4249 }
4250 return replaceOperand(SI, 0, Op0);
4251 }
4252
4253 ConstantInt *SubLHS;
4254 if (match(Cond, m_Sub(m_ConstantInt(SubLHS), m_Value(Op0)))) {
4255 // Change 'switch (1-X) case 1:' into 'switch (X) case 0'.
4256 for (auto Case : SI.cases()) {
4257 Constant *NewCase = ConstantExpr::getSub(SubLHS, Case.getCaseValue());
4258 assert(isa<ConstantInt>(NewCase) &&
4259 "Result of expression should be constant");
4260 Case.setValue(cast<ConstantInt>(NewCase));
4261 }
4262 return replaceOperand(SI, 0, Op0);
4263 }
4264
4265 uint64_t ShiftAmt;
4266 if (match(Cond, m_Shl(m_Value(Op0), m_ConstantInt(ShiftAmt))) &&
4267 ShiftAmt < Op0->getType()->getScalarSizeInBits() &&
4268 all_of(SI.cases(), [&](const auto &Case) {
4269 return Case.getCaseValue()->getValue().countr_zero() >= ShiftAmt;
4270 })) {
4271 // Change 'switch (X << 2) case 4:' into 'switch (X) case 1:'.
4273 if (Shl->hasNoUnsignedWrap() || Shl->hasNoSignedWrap() ||
4274 Shl->hasOneUse()) {
4275 Value *NewCond = Op0;
4276 if (!Shl->hasNoUnsignedWrap() && !Shl->hasNoSignedWrap()) {
4277 // If the shift may wrap, we need to mask off the shifted bits.
4278 unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
4279 NewCond = Builder.CreateAnd(
4280 Op0, APInt::getLowBitsSet(BitWidth, BitWidth - ShiftAmt));
4281 }
4282 for (auto Case : SI.cases()) {
4283 const APInt &CaseVal = Case.getCaseValue()->getValue();
4284 APInt ShiftedCase = Shl->hasNoSignedWrap() ? CaseVal.ashr(ShiftAmt)
4285 : CaseVal.lshr(ShiftAmt);
4286 Case.setValue(ConstantInt::get(SI.getContext(), ShiftedCase));
4287 }
4288 return replaceOperand(SI, 0, NewCond);
4289 }
4290 }
4291
4292 // Fold switch(zext/sext(X)) into switch(X) if possible.
4293 if (match(Cond, m_ZExtOrSExt(m_Value(Op0)))) {
4294 bool IsZExt = isa<ZExtInst>(Cond);
4295 Type *SrcTy = Op0->getType();
4296 unsigned NewWidth = SrcTy->getScalarSizeInBits();
4297
4298 if (all_of(SI.cases(), [&](const auto &Case) {
4299 const APInt &CaseVal = Case.getCaseValue()->getValue();
4300 return IsZExt ? CaseVal.isIntN(NewWidth)
4301 : CaseVal.isSignedIntN(NewWidth);
4302 })) {
4303 for (auto &Case : SI.cases()) {
4304 APInt TruncatedCase = Case.getCaseValue()->getValue().trunc(NewWidth);
4305 Case.setValue(ConstantInt::get(SI.getContext(), TruncatedCase));
4306 }
4307 return replaceOperand(SI, 0, Op0);
4308 }
4309 }
4310
4311 // Fold switch(select cond, X, Y) into switch(X/Y) if possible
4312 if (auto *Select = dyn_cast<SelectInst>(Cond)) {
4313 if (Value *V =
4314 simplifySwitchOnSelectUsingRanges(SI, Select, /*IsTrueArm=*/true))
4315 return replaceOperand(SI, 0, V);
4316 if (Value *V =
4317 simplifySwitchOnSelectUsingRanges(SI, Select, /*IsTrueArm=*/false))
4318 return replaceOperand(SI, 0, V);
4319 }
4320
4321 KnownBits Known = computeKnownBits(Cond, &SI);
4322 unsigned LeadingKnownZeros = Known.countMinLeadingZeros();
4323 unsigned LeadingKnownOnes = Known.countMinLeadingOnes();
4324
4325 // Compute the number of leading bits we can ignore.
4326 // TODO: A better way to determine this would use ComputeNumSignBits().
4327 for (const auto &C : SI.cases()) {
4328 LeadingKnownZeros =
4329 std::min(LeadingKnownZeros, C.getCaseValue()->getValue().countl_zero());
4330 LeadingKnownOnes =
4331 std::min(LeadingKnownOnes, C.getCaseValue()->getValue().countl_one());
4332 }
4333
4334 unsigned NewWidth = Known.getBitWidth() - std::max(LeadingKnownZeros, LeadingKnownOnes);
4335
4336 // Shrink the condition operand if the new type is smaller than the old type.
4337 // But do not shrink to a non-standard type, because backend can't generate
4338 // good code for that yet.
4339 // TODO: We can make it aggressive again after fixing PR39569.
4340 if (NewWidth > 0 && NewWidth < Known.getBitWidth() &&
4341 shouldChangeType(Known.getBitWidth(), NewWidth)) {
4342 IntegerType *Ty = IntegerType::get(SI.getContext(), NewWidth);
4343 Builder.SetInsertPoint(&SI);
4344 Value *NewCond = Builder.CreateTrunc(Cond, Ty, "trunc");
4345
4346 for (auto Case : SI.cases()) {
4347 APInt TruncatedCase = Case.getCaseValue()->getValue().trunc(NewWidth);
4348 Case.setValue(ConstantInt::get(SI.getContext(), TruncatedCase));
4349 }
4350 return replaceOperand(SI, 0, NewCond);
4351 }
4352
4353 if (isa<UndefValue>(Cond)) {
4354 handlePotentiallyDeadSuccessors(SI.getParent(), /*LiveSucc*/ nullptr);
4355 return nullptr;
4356 }
4357 if (auto *CI = dyn_cast<ConstantInt>(Cond)) {
4359 SI.findCaseValue(CI)->getCaseSuccessor());
4360 return nullptr;
4361 }
4362
4363 return nullptr;
4364}
4365
4367InstCombinerImpl::foldExtractOfOverflowIntrinsic(ExtractValueInst &EV) {
4369 if (!WO)
4370 return nullptr;
4371
4372 Intrinsic::ID OvID = WO->getIntrinsicID();
4373 const APInt *C = nullptr;
4374 if (match(WO->getRHS(), m_APIntAllowPoison(C))) {
4375 if (*EV.idx_begin() == 0 && (OvID == Intrinsic::smul_with_overflow ||
4376 OvID == Intrinsic::umul_with_overflow)) {
4377 // extractvalue (any_mul_with_overflow X, -1), 0 --> -X
4378 if (C->isAllOnes())
4379 return BinaryOperator::CreateNeg(WO->getLHS());
4380 // extractvalue (any_mul_with_overflow X, 2^n), 0 --> X << n
4381 if (C->isPowerOf2()) {
4382 return BinaryOperator::CreateShl(
4383 WO->getLHS(),
4384 ConstantInt::get(WO->getLHS()->getType(), C->logBase2()));
4385 }
4386 }
4387 }
4388
4389 // We're extracting from an overflow intrinsic. See if we're the only user.
4390 // That allows us to simplify multiple result intrinsics to simpler things
4391 // that just get one value.
4392 if (!WO->hasOneUse())
4393 return nullptr;
4394
4395 // Check if we're grabbing only the result of a 'with overflow' intrinsic
4396 // and replace it with a traditional binary instruction.
4397 if (*EV.idx_begin() == 0) {
4398 Instruction::BinaryOps BinOp = WO->getBinaryOp();
4399 Value *LHS = WO->getLHS(), *RHS = WO->getRHS();
4400 // Replace the old instruction's uses with poison.
4401 replaceInstUsesWith(*WO, PoisonValue::get(WO->getType()));
4403 return BinaryOperator::Create(BinOp, LHS, RHS);
4404 }
4405
4406 assert(*EV.idx_begin() == 1 && "Unexpected extract index for overflow inst");
4407
4408 // (usub LHS, RHS) overflows when LHS is unsigned-less-than RHS.
4409 if (OvID == Intrinsic::usub_with_overflow)
4410 return new ICmpInst(ICmpInst::ICMP_ULT, WO->getLHS(), WO->getRHS());
4411
4412 // smul with i1 types overflows when both sides are set: -1 * -1 == +1, but
4413 // +1 is not possible because we assume signed values.
4414 if (OvID == Intrinsic::smul_with_overflow &&
4415 WO->getLHS()->getType()->isIntOrIntVectorTy(1))
4416 return BinaryOperator::CreateAnd(WO->getLHS(), WO->getRHS());
4417
4418 // extractvalue (umul_with_overflow X, X), 1 -> X u> 2^(N/2)-1
4419 if (OvID == Intrinsic::umul_with_overflow && WO->getLHS() == WO->getRHS()) {
4420 unsigned BitWidth = WO->getLHS()->getType()->getScalarSizeInBits();
4421 // Only handle even bitwidths for performance reasons.
4422 if (BitWidth % 2 == 0)
4423 return new ICmpInst(
4424 ICmpInst::ICMP_UGT, WO->getLHS(),
4425 ConstantInt::get(WO->getLHS()->getType(),
4427 }
4428
4429 // If only the overflow result is used, and the right hand side is a
4430 // constant (or constant splat), we can remove the intrinsic by directly
4431 // checking for overflow.
4432 if (C) {
4433 // Compute the no-wrap range for LHS given RHS=C, then construct an
4434 // equivalent icmp, potentially using an offset.
4435 ConstantRange NWR = ConstantRange::makeExactNoWrapRegion(
4436 WO->getBinaryOp(), *C, WO->getNoWrapKind());
4437
4438 CmpInst::Predicate Pred;
4439 APInt NewRHSC, Offset;
4440 NWR.getEquivalentICmp(Pred, NewRHSC, Offset);
4441 auto *OpTy = WO->getRHS()->getType();
4442 auto *NewLHS = WO->getLHS();
4443 if (Offset != 0)
4444 NewLHS = Builder.CreateAdd(NewLHS, ConstantInt::get(OpTy, Offset));
4445 return new ICmpInst(ICmpInst::getInversePredicate(Pred), NewLHS,
4446 ConstantInt::get(OpTy, NewRHSC));
4447 }
4448
4449 return nullptr;
4450}
4451
4454 InstCombiner::BuilderTy &Builder) {
4455 // Helper to fold frexp of select to select of frexp.
4456
4457 if (!SelectInst->hasOneUse() || !FrexpCall->hasOneUse())
4458 return nullptr;
4460 Value *TrueVal = SelectInst->getTrueValue();
4461 Value *FalseVal = SelectInst->getFalseValue();
4462
4463 const APFloat *ConstVal = nullptr;
4464 Value *VarOp = nullptr;
4465 bool ConstIsTrue = false;
4466
4467 if (match(TrueVal, m_APFloat(ConstVal))) {
4468 VarOp = FalseVal;
4469 ConstIsTrue = true;
4470 } else if (match(FalseVal, m_APFloat(ConstVal))) {
4471 VarOp = TrueVal;
4472 ConstIsTrue = false;
4473 } else {
4474 return nullptr;
4475 }
4476
4477 Builder.SetInsertPoint(&EV);
4478
4479 CallInst *NewFrexp =
4480 Builder.CreateCall(FrexpCall->getCalledFunction(), {VarOp}, "frexp");
4481 NewFrexp->copyIRFlags(FrexpCall);
4482
4483 Value *NewEV = Builder.CreateExtractValue(NewFrexp, 0, "mantissa");
4484
4485 int Exp;
4486 APFloat Mantissa = frexp(*ConstVal, Exp, APFloat::rmNearestTiesToEven);
4487
4488 Constant *ConstantMantissa = ConstantFP::get(TrueVal->getType(), Mantissa);
4489
4490 Value *NewSel = Builder.CreateSelectFMF(
4491 Cond, ConstIsTrue ? ConstantMantissa : NewEV,
4492 ConstIsTrue ? NewEV : ConstantMantissa, SelectInst, "select.frexp");
4493 return NewSel;
4494}
4496 Value *Agg = EV.getAggregateOperand();
4497
4498 if (!EV.hasIndices())
4499 return replaceInstUsesWith(EV, Agg);
4500
4501 if (Value *V = simplifyExtractValueInst(Agg, EV.getIndices(),
4502 SQ.getWithInstruction(&EV)))
4503 return replaceInstUsesWith(EV, V);
4504
4505 Value *Cond, *TrueVal, *FalseVal;
4507 m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal)))))) {
4508 auto *SelInst =
4509 cast<SelectInst>(cast<IntrinsicInst>(Agg)->getArgOperand(0));
4510 if (Value *Result =
4511 foldFrexpOfSelect(EV, cast<IntrinsicInst>(Agg), SelInst, Builder))
4512 return replaceInstUsesWith(EV, Result);
4513 }
4515 // We're extracting from an insertvalue instruction, compare the indices
4516 const unsigned *exti, *exte, *insi, *inse;
4517 for (exti = EV.idx_begin(), insi = IV->idx_begin(),
4518 exte = EV.idx_end(), inse = IV->idx_end();
4519 exti != exte && insi != inse;
4520 ++exti, ++insi) {
4521 if (*insi != *exti)
4522 // The insert and extract both reference distinctly different elements.
4523 // This means the extract is not influenced by the insert, and we can
4524 // replace the aggregate operand of the extract with the aggregate
4525 // operand of the insert. i.e., replace
4526 // %I = insertvalue { i32, { i32 } } %A, { i32 } { i32 42 }, 1
4527 // %E = extractvalue { i32, { i32 } } %I, 0
4528 // with
4529 // %E = extractvalue { i32, { i32 } } %A, 0
4530 return ExtractValueInst::Create(IV->getAggregateOperand(),
4531 EV.getIndices());
4532 }
4533 if (exti == exte && insi == inse)
4534 // Both iterators are at the end: Index lists are identical. Replace
4535 // %B = insertvalue { i32, { i32 } } %A, i32 42, 1, 0
4536 // %C = extractvalue { i32, { i32 } } %B, 1, 0
4537 // with "i32 42"
4538 return replaceInstUsesWith(EV, IV->getInsertedValueOperand());
4539 if (exti == exte) {
4540 // The extract list is a prefix of the insert list. i.e. replace
4541 // %I = insertvalue { i32, { i32 } } %A, i32 42, 1, 0
4542 // %E = extractvalue { i32, { i32 } } %I, 1
4543 // with
4544 // %X = extractvalue { i32, { i32 } } %A, 1
4545 // %E = insertvalue { i32 } %X, i32 42, 0
4546 // by switching the order of the insert and extract (though the
4547 // insertvalue should be left in, since it may have other uses).
4548 Value *NewEV = Builder.CreateExtractValue(IV->getAggregateOperand(),
4549 EV.getIndices());
4550 return InsertValueInst::Create(NewEV, IV->getInsertedValueOperand(),
4551 ArrayRef(insi, inse));
4552 }
4553 if (insi == inse)
4554 // The insert list is a prefix of the extract list
4555 // We can simply remove the common indices from the extract and make it
4556 // operate on the inserted value instead of the insertvalue result.
4557 // i.e., replace
4558 // %I = insertvalue { i32, { i32 } } %A, { i32 } { i32 42 }, 1
4559 // %E = extractvalue { i32, { i32 } } %I, 1, 0
4560 // with
4561 // %E extractvalue { i32 } { i32 42 }, 0
4562 return ExtractValueInst::Create(IV->getInsertedValueOperand(),
4563 ArrayRef(exti, exte));
4564 }
4565
4566 if (Instruction *R = foldExtractOfOverflowIntrinsic(EV))
4567 return R;
4568
4569 if (LoadInst *L = dyn_cast<LoadInst>(Agg)) {
4570 // Bail out if the aggregate contains scalable vector type
4571 if (auto *STy = dyn_cast<StructType>(Agg->getType());
4572 STy && STy->isScalableTy())
4573 return nullptr;
4574
4575 // If the (non-volatile) load only has one use, we can rewrite this to a
4576 // load from a GEP. This reduces the size of the load. If a load is used
4577 // only by extractvalue instructions then this either must have been
4578 // optimized before, or it is a struct with padding, in which case we
4579 // don't want to do the transformation as it loses padding knowledge.
4580 if (L->isSimple() && L->hasOneUse()) {
4581 // extractvalue has integer indices, getelementptr has Value*s. Convert.
4582 SmallVector<Value*, 4> Indices;
4583 // Prefix an i32 0 since we need the first element.
4584 Indices.push_back(Builder.getInt32(0));
4585 for (unsigned Idx : EV.indices())
4586 Indices.push_back(Builder.getInt32(Idx));
4587
4588 // We need to insert these at the location of the old load, not at that of
4589 // the extractvalue.
4590 Builder.SetInsertPoint(L);
4591 Value *GEP = Builder.CreateInBoundsGEP(L->getType(),
4592 L->getPointerOperand(), Indices);
4593 Instruction *NL = Builder.CreateLoad(EV.getType(), GEP);
4594 // Whatever aliasing information we had for the orignal load must also
4595 // hold for the smaller load, so propagate the annotations.
4596 NL->setAAMetadata(L->getAAMetadata());
4597 // Returning the load directly will cause the main loop to insert it in
4598 // the wrong spot, so use replaceInstUsesWith().
4599 return replaceInstUsesWith(EV, NL);
4600 }
4601 }
4602
4603 if (auto *PN = dyn_cast<PHINode>(Agg))
4604 if (Instruction *Res = foldOpIntoPhi(EV, PN))
4605 return Res;
4606
4607 // Canonicalize extract (select Cond, TV, FV)
4608 // -> select cond, (extract TV), (extract FV)
4609 if (auto *SI = dyn_cast<SelectInst>(Agg))
4610 if (Instruction *R = FoldOpIntoSelect(EV, SI, /*FoldWithMultiUse=*/true))
4611 return R;
4612
4613 // We could simplify extracts from other values. Note that nested extracts may
4614 // already be simplified implicitly by the above: extract (extract (insert) )
4615 // will be translated into extract ( insert ( extract ) ) first and then just
4616 // the value inserted, if appropriate. Similarly for extracts from single-use
4617 // loads: extract (extract (load)) will be translated to extract (load (gep))
4618 // and if again single-use then via load (gep (gep)) to load (gep).
4619 // However, double extracts from e.g. function arguments or return values
4620 // aren't handled yet.
4621 return nullptr;
4622}
4623
4624/// Return 'true' if the given typeinfo will match anything.
4625static bool isCatchAll(EHPersonality Personality, Constant *TypeInfo) {
4626 switch (Personality) {
4630 // The GCC C EH and Rust personality only exists to support cleanups, so
4631 // it's not clear what the semantics of catch clauses are.
4632 return false;
4634 return false;
4636 // While __gnat_all_others_value will match any Ada exception, it doesn't
4637 // match foreign exceptions (or didn't, before gcc-4.7).
4638 return false;
4649 return TypeInfo->isNullValue();
4650 }
4651 llvm_unreachable("invalid enum");
4652}
4653
4654static bool shorter_filter(const Value *LHS, const Value *RHS) {
4655 return
4656 cast<ArrayType>(LHS->getType())->getNumElements()
4657 <
4658 cast<ArrayType>(RHS->getType())->getNumElements();
4659}
4660
4662 // The logic here should be correct for any real-world personality function.
4663 // However if that turns out not to be true, the offending logic can always
4664 // be conditioned on the personality function, like the catch-all logic is.
4665 EHPersonality Personality =
4666 classifyEHPersonality(LI.getParent()->getParent()->getPersonalityFn());
4667
4668 // Simplify the list of clauses, eg by removing repeated catch clauses
4669 // (these are often created by inlining).
4670 bool MakeNewInstruction = false; // If true, recreate using the following:
4671 SmallVector<Constant *, 16> NewClauses; // - Clauses for the new instruction;
4672 bool CleanupFlag = LI.isCleanup(); // - The new instruction is a cleanup.
4673
4674 SmallPtrSet<Value *, 16> AlreadyCaught; // Typeinfos known caught already.
4675 for (unsigned i = 0, e = LI.getNumClauses(); i != e; ++i) {
4676 bool isLastClause = i + 1 == e;
4677 if (LI.isCatch(i)) {
4678 // A catch clause.
4679 Constant *CatchClause = LI.getClause(i);
4680 Constant *TypeInfo = CatchClause->stripPointerCasts();
4681
4682 // If we already saw this clause, there is no point in having a second
4683 // copy of it.
4684 if (AlreadyCaught.insert(TypeInfo).second) {
4685 // This catch clause was not already seen.
4686 NewClauses.push_back(CatchClause);
4687 } else {
4688 // Repeated catch clause - drop the redundant copy.
4689 MakeNewInstruction = true;
4690 }
4691
4692 // If this is a catch-all then there is no point in keeping any following
4693 // clauses or marking the landingpad as having a cleanup.
4694 if (isCatchAll(Personality, TypeInfo)) {
4695 if (!isLastClause)
4696 MakeNewInstruction = true;
4697 CleanupFlag = false;
4698 break;
4699 }
4700 } else {
4701 // A filter clause. If any of the filter elements were already caught
4702 // then they can be dropped from the filter. It is tempting to try to
4703 // exploit the filter further by saying that any typeinfo that does not
4704 // occur in the filter can't be caught later (and thus can be dropped).
4705 // However this would be wrong, since typeinfos can match without being
4706 // equal (for example if one represents a C++ class, and the other some
4707 // class derived from it).
4708 assert(LI.isFilter(i) && "Unsupported landingpad clause!");
4709 Constant *FilterClause = LI.getClause(i);
4710 ArrayType *FilterType = cast<ArrayType>(FilterClause->getType());
4711 unsigned NumTypeInfos = FilterType->getNumElements();
4712
4713 // An empty filter catches everything, so there is no point in keeping any
4714 // following clauses or marking the landingpad as having a cleanup. By
4715 // dealing with this case here the following code is made a bit simpler.
4716 if (!NumTypeInfos) {
4717 NewClauses.push_back(FilterClause);
4718 if (!isLastClause)
4719 MakeNewInstruction = true;
4720 CleanupFlag = false;
4721 break;
4722 }
4723
4724 bool MakeNewFilter = false; // If true, make a new filter.
4725 SmallVector<Constant *, 16> NewFilterElts; // New elements.
4726 if (isa<ConstantAggregateZero>(FilterClause)) {
4727 // Not an empty filter - it contains at least one null typeinfo.
4728 assert(NumTypeInfos > 0 && "Should have handled empty filter already!");
4729 Constant *TypeInfo =
4731 // If this typeinfo is a catch-all then the filter can never match.
4732 if (isCatchAll(Personality, TypeInfo)) {
4733 // Throw the filter away.
4734 MakeNewInstruction = true;
4735 continue;
4736 }
4737
4738 // There is no point in having multiple copies of this typeinfo, so
4739 // discard all but the first copy if there is more than one.
4740 NewFilterElts.push_back(TypeInfo);
4741 if (NumTypeInfos > 1)
4742 MakeNewFilter = true;
4743 } else {
4744 ConstantArray *Filter = cast<ConstantArray>(FilterClause);
4745 SmallPtrSet<Value *, 16> SeenInFilter; // For uniquing the elements.
4746 NewFilterElts.reserve(NumTypeInfos);
4747
4748 // Remove any filter elements that were already caught or that already
4749 // occurred in the filter. While there, see if any of the elements are
4750 // catch-alls. If so, the filter can be discarded.
4751 bool SawCatchAll = false;
4752 for (unsigned j = 0; j != NumTypeInfos; ++j) {
4753 Constant *Elt = Filter->getOperand(j);
4754 Constant *TypeInfo = Elt->stripPointerCasts();
4755 if (isCatchAll(Personality, TypeInfo)) {
4756 // This element is a catch-all. Bail out, noting this fact.
4757 SawCatchAll = true;
4758 break;
4759 }
4760
4761 // Even if we've seen a type in a catch clause, we don't want to
4762 // remove it from the filter. An unexpected type handler may be
4763 // set up for a call site which throws an exception of the same
4764 // type caught. In order for the exception thrown by the unexpected
4765 // handler to propagate correctly, the filter must be correctly
4766 // described for the call site.
4767 //
4768 // Example:
4769 //
4770 // void unexpected() { throw 1;}
4771 // void foo() throw (int) {
4772 // std::set_unexpected(unexpected);
4773 // try {
4774 // throw 2.0;
4775 // } catch (int i) {}
4776 // }
4777
4778 // There is no point in having multiple copies of the same typeinfo in
4779 // a filter, so only add it if we didn't already.
4780 if (SeenInFilter.insert(TypeInfo).second)
4781 NewFilterElts.push_back(cast<Constant>(Elt));
4782 }
4783 // A filter containing a catch-all cannot match anything by definition.
4784 if (SawCatchAll) {
4785 // Throw the filter away.
4786 MakeNewInstruction = true;
4787 continue;
4788 }
4789
4790 // If we dropped something from the filter, make a new one.
4791 if (NewFilterElts.size() < NumTypeInfos)
4792 MakeNewFilter = true;
4793 }
4794 if (MakeNewFilter) {
4795 FilterType = ArrayType::get(FilterType->getElementType(),
4796 NewFilterElts.size());
4797 FilterClause = ConstantArray::get(FilterType, NewFilterElts);
4798 MakeNewInstruction = true;
4799 }
4800
4801 NewClauses.push_back(FilterClause);
4802
4803 // If the new filter is empty then it will catch everything so there is
4804 // no point in keeping any following clauses or marking the landingpad
4805 // as having a cleanup. The case of the original filter being empty was
4806 // already handled above.
4807 if (MakeNewFilter && !NewFilterElts.size()) {
4808 assert(MakeNewInstruction && "New filter but not a new instruction!");
4809 CleanupFlag = false;
4810 break;
4811 }
4812 }
4813 }
4814
4815 // If several filters occur in a row then reorder them so that the shortest
4816 // filters come first (those with the smallest number of elements). This is
4817 // advantageous because shorter filters are more likely to match, speeding up
4818 // unwinding, but mostly because it increases the effectiveness of the other
4819 // filter optimizations below.
4820 for (unsigned i = 0, e = NewClauses.size(); i + 1 < e; ) {
4821 unsigned j;
4822 // Find the maximal 'j' s.t. the range [i, j) consists entirely of filters.
4823 for (j = i; j != e; ++j)
4824 if (!isa<ArrayType>(NewClauses[j]->getType()))
4825 break;
4826
4827 // Check whether the filters are already sorted by length. We need to know
4828 // if sorting them is actually going to do anything so that we only make a
4829 // new landingpad instruction if it does.
4830 for (unsigned k = i; k + 1 < j; ++k)
4831 if (shorter_filter(NewClauses[k+1], NewClauses[k])) {
4832 // Not sorted, so sort the filters now. Doing an unstable sort would be
4833 // correct too but reordering filters pointlessly might confuse users.
4834 std::stable_sort(NewClauses.begin() + i, NewClauses.begin() + j,
4836 MakeNewInstruction = true;
4837 break;
4838 }
4839
4840 // Look for the next batch of filters.
4841 i = j + 1;
4842 }
4843
4844 // If typeinfos matched if and only if equal, then the elements of a filter L
4845 // that occurs later than a filter F could be replaced by the intersection of
4846 // the elements of F and L. In reality two typeinfos can match without being
4847 // equal (for example if one represents a C++ class, and the other some class
4848 // derived from it) so it would be wrong to perform this transform in general.
4849 // However the transform is correct and useful if F is a subset of L. In that
4850 // case L can be replaced by F, and thus removed altogether since repeating a
4851 // filter is pointless. So here we look at all pairs of filters F and L where
4852 // L follows F in the list of clauses, and remove L if every element of F is
4853 // an element of L. This can occur when inlining C++ functions with exception
4854 // specifications.
4855 for (unsigned i = 0; i + 1 < NewClauses.size(); ++i) {
4856 // Examine each filter in turn.
4857 Value *Filter = NewClauses[i];
4858 ArrayType *FTy = dyn_cast<ArrayType>(Filter->getType());
4859 if (!FTy)
4860 // Not a filter - skip it.
4861 continue;
4862 unsigned FElts = FTy->getNumElements();
4863 // Examine each filter following this one. Doing this backwards means that
4864 // we don't have to worry about filters disappearing under us when removed.
4865 for (unsigned j = NewClauses.size() - 1; j != i; --j) {
4866 Value *LFilter = NewClauses[j];
4867 ArrayType *LTy = dyn_cast<ArrayType>(LFilter->getType());
4868 if (!LTy)
4869 // Not a filter - skip it.
4870 continue;
4871 // If Filter is a subset of LFilter, i.e. every element of Filter is also
4872 // an element of LFilter, then discard LFilter.
4873 SmallVectorImpl<Constant *>::iterator J = NewClauses.begin() + j;
4874 // If Filter is empty then it is a subset of LFilter.
4875 if (!FElts) {
4876 // Discard LFilter.
4877 NewClauses.erase(J);
4878 MakeNewInstruction = true;
4879 // Move on to the next filter.
4880 continue;
4881 }
4882 unsigned LElts = LTy->getNumElements();
4883 // If Filter is longer than LFilter then it cannot be a subset of it.
4884 if (FElts > LElts)
4885 // Move on to the next filter.
4886 continue;
4887 // At this point we know that LFilter has at least one element.
4888 if (isa<ConstantAggregateZero>(LFilter)) { // LFilter only contains zeros.
4889 // Filter is a subset of LFilter iff Filter contains only zeros (as we
4890 // already know that Filter is not longer than LFilter).
4892 assert(FElts <= LElts && "Should have handled this case earlier!");
4893 // Discard LFilter.
4894 NewClauses.erase(J);
4895 MakeNewInstruction = true;
4896 }
4897 // Move on to the next filter.
4898 continue;
4899 }
4900 ConstantArray *LArray = cast<ConstantArray>(LFilter);
4901 if (isa<ConstantAggregateZero>(Filter)) { // Filter only contains zeros.
4902 // Since Filter is non-empty and contains only zeros, it is a subset of
4903 // LFilter iff LFilter contains a zero.
4904 assert(FElts > 0 && "Should have eliminated the empty filter earlier!");
4905 for (unsigned l = 0; l != LElts; ++l)
4906 if (LArray->getOperand(l)->isNullValue()) {
4907 // LFilter contains a zero - discard it.
4908 NewClauses.erase(J);
4909 MakeNewInstruction = true;
4910 break;
4911 }
4912 // Move on to the next filter.
4913 continue;
4914 }
4915 // At this point we know that both filters are ConstantArrays. Loop over
4916 // operands to see whether every element of Filter is also an element of
4917 // LFilter. Since filters tend to be short this is probably faster than
4918 // using a method that scales nicely.
4920 bool AllFound = true;
4921 for (unsigned f = 0; f != FElts; ++f) {
4922 Value *FTypeInfo = FArray->getOperand(f)->stripPointerCasts();
4923 AllFound = false;
4924 for (unsigned l = 0; l != LElts; ++l) {
4925 Value *LTypeInfo = LArray->getOperand(l)->stripPointerCasts();
4926 if (LTypeInfo == FTypeInfo) {
4927 AllFound = true;
4928 break;
4929 }
4930 }
4931 if (!AllFound)
4932 break;
4933 }
4934 if (AllFound) {
4935 // Discard LFilter.
4936 NewClauses.erase(J);
4937 MakeNewInstruction = true;
4938 }
4939 // Move on to the next filter.
4940 }
4941 }
4942
4943 // If we changed any of the clauses, replace the old landingpad instruction
4944 // with a new one.
4945 if (MakeNewInstruction) {
4947 NewClauses.size());
4948 for (Constant *C : NewClauses)
4949 NLI->addClause(C);
4950 // A landing pad with no clauses must have the cleanup flag set. It is
4951 // theoretically possible, though highly unlikely, that we eliminated all
4952 // clauses. If so, force the cleanup flag to true.
4953 if (NewClauses.empty())
4954 CleanupFlag = true;
4955 NLI->setCleanup(CleanupFlag);
4956 return NLI;
4957 }
4958
4959 // Even if none of the clauses changed, we may nonetheless have understood
4960 // that the cleanup flag is pointless. Clear it if so.
4961 if (LI.isCleanup() != CleanupFlag) {
4962 assert(!CleanupFlag && "Adding a cleanup, not removing one?!");
4963 LI.setCleanup(CleanupFlag);
4964 return &LI;
4965 }
4966
4967 return nullptr;
4968}
4969
4970Value *
4972 // Try to push freeze through instructions that propagate but don't produce
4973 // poison as far as possible. If an operand of freeze does not produce poison
4974 // then push the freeze through to the operands that are not guaranteed
4975 // non-poison. The actual transform is as follows.
4976 // Op1 = ... ; Op1 can be poison
4977 // Op0 = Inst(Op1, NonPoisonOps...)
4978 // ... = Freeze(Op0)
4979 // =>
4980 // Op1 = ...
4981 // Op1.fr = Freeze(Op1)
4982 // ... = Inst(Op1.fr, NonPoisonOps...)
4983
4984 auto CanPushFreeze = [](Value *V) {
4985 if (!isa<Instruction>(V) || isa<PHINode>(V))
4986 return false;
4987
4988 // We can't push the freeze through an instruction which can itself create
4989 // poison. If the only source of new poison is flags, we can simply
4990 // strip them (since we know the only use is the freeze and nothing can
4991 // benefit from them.)
4993 /*ConsiderFlagsAndMetadata*/ false);
4994 };
4995
4996 // Pushing freezes up long instruction chains can be expensive. Instead,
4997 // we directly push the freeze all the way to the leaves. However, we leave
4998 // deduplication of freezes on the same value for freezeOtherUses().
4999 Use *OrigUse = &OrigFI.getOperandUse(0);
5002 Worklist.push_back(OrigUse);
5003 while (!Worklist.empty()) {
5004 auto *U = Worklist.pop_back_val();
5005 Value *V = U->get();
5006 if (!CanPushFreeze(V)) {
5007 // If we can't push through the original instruction, abort the transform.
5008 if (U == OrigUse)
5009 return nullptr;
5010
5011 auto *UserI = cast<Instruction>(U->getUser());
5012 Builder.SetInsertPoint(UserI);
5013 Value *Frozen = Builder.CreateFreeze(V, V->getName() + ".fr");
5014 U->set(Frozen);
5015 continue;
5016 }
5017
5018 auto *I = cast<Instruction>(V);
5019 if (!Visited.insert(I).second)
5020 continue;
5021
5022 // reverse() to emit freezes in a more natural order.
5023 for (Use &Op : reverse(I->operands())) {
5024 Value *OpV = Op.get();
5026 continue;
5027 Worklist.push_back(&Op);
5028 }
5029
5030 I->dropPoisonGeneratingAnnotations();
5031 this->Worklist.add(I);
5032 }
5033
5034 return OrigUse->get();
5035}
5036
5038 PHINode *PN) {
5039 // Detect whether this is a recurrence with a start value and some number of
5040 // backedge values. We'll check whether we can push the freeze through the
5041 // backedge values (possibly dropping poison flags along the way) until we
5042 // reach the phi again. In that case, we can move the freeze to the start
5043 // value.
5044 Use *StartU = nullptr;
5046 for (Use &U : PN->incoming_values()) {
5047 if (DT.dominates(PN->getParent(), PN->getIncomingBlock(U))) {
5048 // Add backedge value to worklist.
5049 Worklist.push_back(U.get());
5050 continue;
5051 }
5052
5053 // Don't bother handling multiple start values.
5054 if (StartU)
5055 return nullptr;
5056 StartU = &U;
5057 }
5058
5059 if (!StartU || Worklist.empty())
5060 return nullptr; // Not a recurrence.
5061
5062 Value *StartV = StartU->get();
5063 BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
5064 bool StartNeedsFreeze = !isGuaranteedNotToBeUndefOrPoison(StartV);
5065 // We can't insert freeze if the start value is the result of the
5066 // terminator (e.g. an invoke).
5067 if (StartNeedsFreeze && StartBB->getTerminator() == StartV)
5068 return nullptr;
5069
5072 while (!Worklist.empty()) {
5073 Value *V = Worklist.pop_back_val();
5074 if (!Visited.insert(V).second)
5075 continue;
5076
5077 if (Visited.size() > 32)
5078 return nullptr; // Limit the total number of values we inspect.
5079
5080 // Assume that PN is non-poison, because it will be after the transform.
5081 if (V == PN || isGuaranteedNotToBeUndefOrPoison(V))
5082 continue;
5083
5086 /*ConsiderFlagsAndMetadata*/ false))
5087 return nullptr;
5088
5089 DropFlags.push_back(I);
5090 append_range(Worklist, I->operands());
5091 }
5092
5093 for (Instruction *I : DropFlags)
5094 I->dropPoisonGeneratingAnnotations();
5095
5096 if (StartNeedsFreeze) {
5097 Builder.SetInsertPoint(StartBB->getTerminator());
5098 Value *FrozenStartV = Builder.CreateFreeze(StartV,
5099 StartV->getName() + ".fr");
5100 replaceUse(*StartU, FrozenStartV);
5101 }
5102 return replaceInstUsesWith(FI, PN);
5103}
5104
5106 Value *Op = FI.getOperand(0);
5107
5108 if (isa<Constant>(Op) || Op->hasOneUse())
5109 return false;
5110
5111 // Move the freeze directly after the definition of its operand, so that
5112 // it dominates the maximum number of uses. Note that it may not dominate
5113 // *all* uses if the operand is an invoke/callbr and the use is in a phi on
5114 // the normal/default destination. This is why the domination check in the
5115 // replacement below is still necessary.
5116 BasicBlock::iterator MoveBefore;
5117 if (isa<Argument>(Op)) {
5118 MoveBefore =
5120 } else {
5121 auto MoveBeforeOpt = cast<Instruction>(Op)->getInsertionPointAfterDef();
5122 if (!MoveBeforeOpt)
5123 return false;
5124 MoveBefore = *MoveBeforeOpt;
5125 }
5126
5127 // Re-point iterator to come after any debug-info records.
5128 MoveBefore.setHeadBit(false);
5129
5130 bool Changed = false;
5131 if (&FI != &*MoveBefore) {
5132 FI.moveBefore(*MoveBefore->getParent(), MoveBefore);
5133 Changed = true;
5134 }
5135
5136 Op->replaceUsesWithIf(&FI, [&](Use &U) -> bool {
5137 bool Dominates = DT.dominates(&FI, U);
5138 Changed |= Dominates;
5139 return Dominates;
5140 });
5141
5142 return Changed;
5143}
5144
5145// Check if any direct or bitcast user of this value is a shuffle instruction.
5147 for (auto *U : V->users()) {
5149 return true;
5150 else if (match(U, m_BitCast(m_Specific(V))) && isUsedWithinShuffleVector(U))
5151 return true;
5152 }
5153 return false;
5154}
5155
5157 Value *Op0 = I.getOperand(0);
5158
5159 if (Value *V = simplifyFreezeInst(Op0, SQ.getWithInstruction(&I)))
5160 return replaceInstUsesWith(I, V);
5161
5162 // freeze (phi const, x) --> phi const, (freeze x)
5163 if (auto *PN = dyn_cast<PHINode>(Op0)) {
5164 if (Instruction *NV = foldOpIntoPhi(I, PN))
5165 return NV;
5166 if (Instruction *NV = foldFreezeIntoRecurrence(I, PN))
5167 return NV;
5168 }
5169
5171 return replaceInstUsesWith(I, NI);
5172
5173 // If I is freeze(undef), check its uses and fold it to a fixed constant.
5174 // - or: pick -1
5175 // - select's condition: if the true value is constant, choose it by making
5176 // the condition true.
5177 // - phi: pick the common constant across operands
5178 // - default: pick 0
5179 //
5180 // Note that this transform is intentionally done here rather than
5181 // via an analysis in InstSimplify or at individual user sites. That is
5182 // because we must produce the same value for all uses of the freeze -
5183 // it's the reason "freeze" exists!
5184 //
5185 // TODO: This could use getBinopAbsorber() / getBinopIdentity() to avoid
5186 // duplicating logic for binops at least.
5187 auto getUndefReplacement = [&](Type *Ty) {
5188 auto pickCommonConstantFromPHI = [](PHINode &PN) -> Value * {
5189 // phi(freeze(undef), C, C). Choose C for freeze so the PHI can be
5190 // removed.
5191 Constant *BestValue = nullptr;
5192 for (Value *V : PN.incoming_values()) {
5193 if (match(V, m_Freeze(m_Undef())))
5194 continue;
5195
5197 if (!C)
5198 return nullptr;
5199
5201 return nullptr;
5202
5203 if (BestValue && BestValue != C)
5204 return nullptr;
5205
5206 BestValue = C;
5207 }
5208 return BestValue;
5209 };
5210
5211 Value *NullValue = Constant::getNullValue(Ty);
5212 Value *BestValue = nullptr;
5213 for (auto *U : I.users()) {
5214 Value *V = NullValue;
5215 if (match(U, m_Or(m_Value(), m_Value())))
5217 else if (match(U, m_Select(m_Specific(&I), m_Constant(), m_Value())))
5218 V = ConstantInt::getTrue(Ty);
5219 else if (match(U, m_c_Select(m_Specific(&I), m_Value(V)))) {
5220 if (V == &I || !isGuaranteedNotToBeUndefOrPoison(V, &AC, &I, &DT))
5221 V = NullValue;
5222 } else if (auto *PHI = dyn_cast<PHINode>(U)) {
5223 if (Value *MaybeV = pickCommonConstantFromPHI(*PHI))
5224 V = MaybeV;
5225 }
5226
5227 if (!BestValue)
5228 BestValue = V;
5229 else if (BestValue != V)
5230 BestValue = NullValue;
5231 }
5232 assert(BestValue && "Must have at least one use");
5233 assert(BestValue != &I && "Cannot replace with itself");
5234 return BestValue;
5235 };
5236
5237 if (match(Op0, m_Undef())) {
5238 // Don't fold freeze(undef/poison) if it's used as a vector operand in
5239 // a shuffle. This may improve codegen for shuffles that allow
5240 // unspecified inputs.
5242 return nullptr;
5243 return replaceInstUsesWith(I, getUndefReplacement(I.getType()));
5244 }
5245
5246 auto getFreezeVectorReplacement = [](Constant *C) -> Constant * {
5247 Type *Ty = C->getType();
5248 auto *VTy = dyn_cast<FixedVectorType>(Ty);
5249 if (!VTy)
5250 return nullptr;
5251 unsigned NumElts = VTy->getNumElements();
5252 Constant *BestValue = Constant::getNullValue(VTy->getScalarType());
5253 for (unsigned i = 0; i != NumElts; ++i) {
5254 Constant *EltC = C->getAggregateElement(i);
5255 if (EltC && !match(EltC, m_Undef())) {
5256 BestValue = EltC;
5257 break;
5258 }
5259 }
5260 return Constant::replaceUndefsWith(C, BestValue);
5261 };
5262
5263 Constant *C;
5264 if (match(Op0, m_Constant(C)) && C->containsUndefOrPoisonElement() &&
5265 !C->containsConstantExpression()) {
5266 if (Constant *Repl = getFreezeVectorReplacement(C))
5267 return replaceInstUsesWith(I, Repl);
5268 }
5269
5270 // Replace uses of Op with freeze(Op).
5271 if (freezeOtherUses(I))
5272 return &I;
5273
5274 return nullptr;
5275}
5276
5277/// Check for case where the call writes to an otherwise dead alloca. This
5278/// shows up for unused out-params in idiomatic C/C++ code. Note that this
5279/// helper *only* analyzes the write; doesn't check any other legality aspect.
5281 auto *CB = dyn_cast<CallBase>(I);
5282 if (!CB)
5283 // TODO: handle e.g. store to alloca here - only worth doing if we extend
5284 // to allow reload along used path as described below. Otherwise, this
5285 // is simply a store to a dead allocation which will be removed.
5286 return false;
5287 std::optional<MemoryLocation> Dest = MemoryLocation::getForDest(CB, TLI);
5288 if (!Dest)
5289 return false;
5290 auto *AI = dyn_cast<AllocaInst>(getUnderlyingObject(Dest->Ptr));
5291 if (!AI)
5292 // TODO: allow malloc?
5293 return false;
5294 // TODO: allow memory access dominated by move point? Note that since AI
5295 // could have a reference to itself captured by the call, we would need to
5296 // account for cycles in doing so.
5297 SmallVector<const User *> AllocaUsers;
5299 auto pushUsers = [&](const Instruction &I) {
5300 for (const User *U : I.users()) {
5301 if (Visited.insert(U).second)
5302 AllocaUsers.push_back(U);
5303 }
5304 };
5305 pushUsers(*AI);
5306 while (!AllocaUsers.empty()) {
5307 auto *UserI = cast<Instruction>(AllocaUsers.pop_back_val());
5308 if (isa<GetElementPtrInst>(UserI) || isa<AddrSpaceCastInst>(UserI)) {
5309 pushUsers(*UserI);
5310 continue;
5311 }
5312 if (UserI == CB)
5313 continue;
5314 // TODO: support lifetime.start/end here
5315 return false;
5316 }
5317 return true;
5318}
5319
5320/// Try to move the specified instruction from its current block into the
5321/// beginning of DestBlock, which can only happen if it's safe to move the
5322/// instruction past all of the instructions between it and the end of its
5323/// block.
5325 BasicBlock *DestBlock) {
5326 BasicBlock *SrcBlock = I->getParent();
5327
5328 // Cannot move control-flow-involving, volatile loads, vaarg, etc.
5329 if (isa<PHINode>(I) || I->isEHPad() || I->mayThrow() || !I->willReturn() ||
5330 I->isTerminator())
5331 return false;
5332
5333 // Do not sink static or dynamic alloca instructions. Static allocas must
5334 // remain in the entry block, and dynamic allocas must not be sunk in between
5335 // a stacksave / stackrestore pair, which would incorrectly shorten its
5336 // lifetime.
5337 if (isa<AllocaInst>(I))
5338 return false;
5339
5340 // Do not sink into catchswitch blocks.
5341 if (isa<CatchSwitchInst>(DestBlock->getTerminator()))
5342 return false;
5343
5344 // Do not sink convergent call instructions.
5345 if (auto *CI = dyn_cast<CallInst>(I)) {
5346 if (CI->isConvergent())
5347 return false;
5348 }
5349
5350 // Unless we can prove that the memory write isn't visibile except on the
5351 // path we're sinking to, we must bail.
5352 if (I->mayWriteToMemory()) {
5353 if (!SoleWriteToDeadLocal(I, TLI))
5354 return false;
5355 }
5356
5357 // We can only sink load instructions if there is nothing between the load and
5358 // the end of block that could change the value.
5359 if (I->mayReadFromMemory() &&
5360 !I->hasMetadata(LLVMContext::MD_invariant_load)) {
5361 // We don't want to do any sophisticated alias analysis, so we only check
5362 // the instructions after I in I's parent block if we try to sink to its
5363 // successor block.
5364 if (DestBlock->getUniquePredecessor() != I->getParent())
5365 return false;
5366 for (BasicBlock::iterator Scan = std::next(I->getIterator()),
5367 E = I->getParent()->end();
5368 Scan != E; ++Scan)
5369 if (Scan->mayWriteToMemory())
5370 return false;
5371 }
5372
5373 I->dropDroppableUses([&](const Use *U) {
5374 auto *I = dyn_cast<Instruction>(U->getUser());
5375 if (I && I->getParent() != DestBlock) {
5376 Worklist.add(I);
5377 return true;
5378 }
5379 return false;
5380 });
5381 /// FIXME: We could remove droppable uses that are not dominated by
5382 /// the new position.
5383
5384 BasicBlock::iterator InsertPos = DestBlock->getFirstInsertionPt();
5385 I->moveBefore(*DestBlock, InsertPos);
5386 ++NumSunkInst;
5387
5388 // Also sink all related debug uses from the source basic block. Otherwise we
5389 // get debug use before the def. Attempt to salvage debug uses first, to
5390 // maximise the range variables have location for. If we cannot salvage, then
5391 // mark the location undef: we know it was supposed to receive a new location
5392 // here, but that computation has been sunk.
5393 SmallVector<DbgVariableRecord *, 2> DbgVariableRecords;
5394 findDbgUsers(I, DbgVariableRecords);
5395 if (!DbgVariableRecords.empty())
5396 tryToSinkInstructionDbgVariableRecords(I, InsertPos, SrcBlock, DestBlock,
5397 DbgVariableRecords);
5398
5399 // PS: there are numerous flaws with this behaviour, not least that right now
5400 // assignments can be re-ordered past other assignments to the same variable
5401 // if they use different Values. Creating more undef assignements can never be
5402 // undone. And salvaging all users outside of this block can un-necessarily
5403 // alter the lifetime of the live-value that the variable refers to.
5404 // Some of these things can be resolved by tolerating debug use-before-defs in
5405 // LLVM-IR, however it depends on the instruction-referencing CodeGen backend
5406 // being used for more architectures.
5407
5408 return true;
5409}
5410
5412 Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
5413 BasicBlock *DestBlock,
5414 SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords) {
5415 // For all debug values in the destination block, the sunk instruction
5416 // will still be available, so they do not need to be dropped.
5417
5418 // Fetch all DbgVariableRecords not already in the destination.
5419 SmallVector<DbgVariableRecord *, 2> DbgVariableRecordsToSalvage;
5420 for (auto &DVR : DbgVariableRecords)
5421 if (DVR->getParent() != DestBlock)
5422 DbgVariableRecordsToSalvage.push_back(DVR);
5423
5424 // Fetch a second collection, of DbgVariableRecords in the source block that
5425 // we're going to sink.
5426 SmallVector<DbgVariableRecord *> DbgVariableRecordsToSink;
5427 for (DbgVariableRecord *DVR : DbgVariableRecordsToSalvage)
5428 if (DVR->getParent() == SrcBlock)
5429 DbgVariableRecordsToSink.push_back(DVR);
5430
5431 // Sort DbgVariableRecords according to their position in the block. This is a
5432 // partial order: DbgVariableRecords attached to different instructions will
5433 // be ordered by the instruction order, but DbgVariableRecords attached to the
5434 // same instruction won't have an order.
5435 auto Order = [](DbgVariableRecord *A, DbgVariableRecord *B) -> bool {
5436 return B->getInstruction()->comesBefore(A->getInstruction());
5437 };
5438 llvm::stable_sort(DbgVariableRecordsToSink, Order);
5439
5440 // If there are two assignments to the same variable attached to the same
5441 // instruction, the ordering between the two assignments is important. Scan
5442 // for this (rare) case and establish which is the last assignment.
5443 using InstVarPair = std::pair<const Instruction *, DebugVariable>;
5445 if (DbgVariableRecordsToSink.size() > 1) {
5447 // Count how many assignments to each variable there is per instruction.
5448 for (DbgVariableRecord *DVR : DbgVariableRecordsToSink) {
5449 DebugVariable DbgUserVariable =
5450 DebugVariable(DVR->getVariable(), DVR->getExpression(),
5451 DVR->getDebugLoc()->getInlinedAt());
5452 CountMap[std::make_pair(DVR->getInstruction(), DbgUserVariable)] += 1;
5453 }
5454
5455 // If there are any instructions with two assignments, add them to the
5456 // FilterOutMap to record that they need extra filtering.
5458 for (auto It : CountMap) {
5459 if (It.second > 1) {
5460 FilterOutMap[It.first] = nullptr;
5461 DupSet.insert(It.first.first);
5462 }
5463 }
5464
5465 // For all instruction/variable pairs needing extra filtering, find the
5466 // latest assignment.
5467 for (const Instruction *Inst : DupSet) {
5468 for (DbgVariableRecord &DVR :
5469 llvm::reverse(filterDbgVars(Inst->getDbgRecordRange()))) {
5470 DebugVariable DbgUserVariable =
5471 DebugVariable(DVR.getVariable(), DVR.getExpression(),
5472 DVR.getDebugLoc()->getInlinedAt());
5473 auto FilterIt =
5474 FilterOutMap.find(std::make_pair(Inst, DbgUserVariable));
5475 if (FilterIt == FilterOutMap.end())
5476 continue;
5477 if (FilterIt->second != nullptr)
5478 continue;
5479 FilterIt->second = &DVR;
5480 }
5481 }
5482 }
5483
5484 // Perform cloning of the DbgVariableRecords that we plan on sinking, filter
5485 // out any duplicate assignments identified above.
5487 SmallSet<DebugVariable, 4> SunkVariables;
5488 for (DbgVariableRecord *DVR : DbgVariableRecordsToSink) {
5490 continue;
5491
5492 DebugVariable DbgUserVariable =
5493 DebugVariable(DVR->getVariable(), DVR->getExpression(),
5494 DVR->getDebugLoc()->getInlinedAt());
5495
5496 // For any variable where there were multiple assignments in the same place,
5497 // ignore all but the last assignment.
5498 if (!FilterOutMap.empty()) {
5499 InstVarPair IVP = std::make_pair(DVR->getInstruction(), DbgUserVariable);
5500 auto It = FilterOutMap.find(IVP);
5501
5502 // Filter out.
5503 if (It != FilterOutMap.end() && It->second != DVR)
5504 continue;
5505 }
5506
5507 if (!SunkVariables.insert(DbgUserVariable).second)
5508 continue;
5509
5510 if (DVR->isDbgAssign())
5511 continue;
5512
5513 DVRClones.emplace_back(DVR->clone());
5514 LLVM_DEBUG(dbgs() << "CLONE: " << *DVRClones.back() << '\n');
5515 }
5516
5517 // Perform salvaging without the clones, then sink the clones.
5518 if (DVRClones.empty())
5519 return;
5520
5521 salvageDebugInfoForDbgValues(*I, DbgVariableRecordsToSalvage);
5522
5523 // The clones are in reverse order of original appearance. Assert that the
5524 // head bit is set on the iterator as we _should_ have received it via
5525 // getFirstInsertionPt. Inserting like this will reverse the clone order as
5526 // we'll repeatedly insert at the head, such as:
5527 // DVR-3 (third insertion goes here)
5528 // DVR-2 (second insertion goes here)
5529 // DVR-1 (first insertion goes here)
5530 // Any-Prior-DVRs
5531 // InsertPtInst
5532 assert(InsertPos.getHeadBit());
5533 for (DbgVariableRecord *DVRClone : DVRClones) {
5534 InsertPos->getParent()->insertDbgRecordBefore(DVRClone, InsertPos);
5535 LLVM_DEBUG(dbgs() << "SINK: " << *DVRClone << '\n');
5536 }
5537}
5538
5540 while (!Worklist.isEmpty()) {
5541 // Walk deferred instructions in reverse order, and push them to the
5542 // worklist, which means they'll end up popped from the worklist in-order.
5543 while (Instruction *I = Worklist.popDeferred()) {
5544 // Check to see if we can DCE the instruction. We do this already here to
5545 // reduce the number of uses and thus allow other folds to trigger.
5546 // Note that eraseInstFromFunction() may push additional instructions on
5547 // the deferred worklist, so this will DCE whole instruction chains.
5550 ++NumDeadInst;
5551 continue;
5552 }
5553
5554 Worklist.push(I);
5555 }
5556
5557 Instruction *I = Worklist.removeOne();
5558 if (I == nullptr) continue; // skip null values.
5559
5560 // Check to see if we can DCE the instruction.
5563 ++NumDeadInst;
5564 continue;
5565 }
5566
5567 if (!DebugCounter::shouldExecute(VisitCounter))
5568 continue;
5569
5570 // See if we can trivially sink this instruction to its user if we can
5571 // prove that the successor is not executed more frequently than our block.
5572 // Return the UserBlock if successful.
5573 auto getOptionalSinkBlockForInst =
5574 [this](Instruction *I) -> std::optional<BasicBlock *> {
5575 if (!EnableCodeSinking)
5576 return std::nullopt;
5577
5578 BasicBlock *BB = I->getParent();
5579 BasicBlock *UserParent = nullptr;
5580 unsigned NumUsers = 0;
5581
5582 for (Use &U : I->uses()) {
5583 User *User = U.getUser();
5584 if (User->isDroppable())
5585 continue;
5586 if (NumUsers > MaxSinkNumUsers)
5587 return std::nullopt;
5588
5589 Instruction *UserInst = cast<Instruction>(User);
5590 // Special handling for Phi nodes - get the block the use occurs in.
5591 BasicBlock *UserBB = UserInst->getParent();
5592 if (PHINode *PN = dyn_cast<PHINode>(UserInst))
5593 UserBB = PN->getIncomingBlock(U);
5594 // Bail out if we have uses in different blocks. We don't do any
5595 // sophisticated analysis (i.e finding NearestCommonDominator of these
5596 // use blocks).
5597 if (UserParent && UserParent != UserBB)
5598 return std::nullopt;
5599 UserParent = UserBB;
5600
5601 // Make sure these checks are done only once, naturally we do the checks
5602 // the first time we get the userparent, this will save compile time.
5603 if (NumUsers == 0) {
5604 // Try sinking to another block. If that block is unreachable, then do
5605 // not bother. SimplifyCFG should handle it.
5606 if (UserParent == BB || !DT.isReachableFromEntry(UserParent))
5607 return std::nullopt;
5608
5609 auto *Term = UserParent->getTerminator();
5610 // See if the user is one of our successors that has only one
5611 // predecessor, so that we don't have to split the critical edge.
5612 // Another option where we can sink is a block that ends with a
5613 // terminator that does not pass control to other block (such as
5614 // return or unreachable or resume). In this case:
5615 // - I dominates the User (by SSA form);
5616 // - the User will be executed at most once.
5617 // So sinking I down to User is always profitable or neutral.
5618 if (UserParent->getUniquePredecessor() != BB && !succ_empty(Term))
5619 return std::nullopt;
5620
5621 assert(DT.dominates(BB, UserParent) && "Dominance relation broken?");
5622 }
5623
5624 NumUsers++;
5625 }
5626
5627 // No user or only has droppable users.
5628 if (!UserParent)
5629 return std::nullopt;
5630
5631 return UserParent;
5632 };
5633
5634 auto OptBB = getOptionalSinkBlockForInst(I);
5635 if (OptBB) {
5636 auto *UserParent = *OptBB;
5637 // Okay, the CFG is simple enough, try to sink this instruction.
5638 if (tryToSinkInstruction(I, UserParent)) {
5639 LLVM_DEBUG(dbgs() << "IC: Sink: " << *I << '\n');
5640 MadeIRChange = true;
5641 // We'll add uses of the sunk instruction below, but since
5642 // sinking can expose opportunities for it's *operands* add
5643 // them to the worklist
5644 for (Use &U : I->operands())
5645 if (Instruction *OpI = dyn_cast<Instruction>(U.get()))
5646 Worklist.push(OpI);
5647 }
5648 }
5649
5650 // Now that we have an instruction, try combining it to simplify it.
5651 Builder.SetInsertPoint(I);
5652 Builder.CollectMetadataToCopy(
5653 I, {LLVMContext::MD_dbg, LLVMContext::MD_annotation});
5654
5655#ifndef NDEBUG
5656 std::string OrigI;
5657#endif
5658 LLVM_DEBUG(raw_string_ostream SS(OrigI); I->print(SS););
5659 LLVM_DEBUG(dbgs() << "IC: Visiting: " << OrigI << '\n');
5660
5661 if (Instruction *Result = visit(*I)) {
5662 ++NumCombined;
5663 // Should we replace the old instruction with a new one?
5664 if (Result != I) {
5665 LLVM_DEBUG(dbgs() << "IC: Old = " << *I << '\n'
5666 << " New = " << *Result << '\n');
5667
5668 // We copy the old instruction's DebugLoc to the new instruction, unless
5669 // InstCombine already assigned a DebugLoc to it, in which case we
5670 // should trust the more specifically selected DebugLoc.
5671 Result->setDebugLoc(Result->getDebugLoc().orElse(I->getDebugLoc()));
5672 // We also copy annotation metadata to the new instruction.
5673 Result->copyMetadata(*I, LLVMContext::MD_annotation);
5674 // Everything uses the new instruction now.
5675 I->replaceAllUsesWith(Result);
5676
5677 // Move the name to the new instruction first.
5678 Result->takeName(I);
5679
5680 // Insert the new instruction into the basic block...
5681 BasicBlock *InstParent = I->getParent();
5682 BasicBlock::iterator InsertPos = I->getIterator();
5683
5684 // Are we replace a PHI with something that isn't a PHI, or vice versa?
5685 if (isa<PHINode>(Result) != isa<PHINode>(I)) {
5686 // We need to fix up the insertion point.
5687 if (isa<PHINode>(I)) // PHI -> Non-PHI
5688 InsertPos = InstParent->getFirstInsertionPt();
5689 else // Non-PHI -> PHI
5690 InsertPos = InstParent->getFirstNonPHIIt();
5691 }
5692
5693 Result->insertInto(InstParent, InsertPos);
5694
5695 // Push the new instruction and any users onto the worklist.
5696 Worklist.pushUsersToWorkList(*Result);
5697 Worklist.push(Result);
5698
5700 } else {
5701 LLVM_DEBUG(dbgs() << "IC: Mod = " << OrigI << '\n'
5702 << " New = " << *I << '\n');
5703
5704 // If the instruction was modified, it's possible that it is now dead.
5705 // if so, remove it.
5708 } else {
5709 Worklist.pushUsersToWorkList(*I);
5710 Worklist.push(I);
5711 }
5712 }
5713 MadeIRChange = true;
5714 }
5715 }
5716
5717 Worklist.zap();
5718 return MadeIRChange;
5719}
5720
5721// Track the scopes used by !alias.scope and !noalias. In a function, a
5722// @llvm.experimental.noalias.scope.decl is only useful if that scope is used
5723// by both sets. If not, the declaration of the scope can be safely omitted.
5724// The MDNode of the scope can be omitted as well for the instructions that are
5725// part of this function. We do not do that at this point, as this might become
5726// too time consuming to do.
5728 SmallPtrSet<const MDNode *, 8> UsedAliasScopesAndLists;
5729 SmallPtrSet<const MDNode *, 8> UsedNoAliasScopesAndLists;
5730
5731public:
5733 // This seems to be faster than checking 'mayReadOrWriteMemory()'.
5734 if (!I->hasMetadataOtherThanDebugLoc())
5735 return;
5736
5737 auto Track = [](Metadata *ScopeList, auto &Container) {
5738 const auto *MDScopeList = dyn_cast_or_null<MDNode>(ScopeList);
5739 if (!MDScopeList || !Container.insert(MDScopeList).second)
5740 return;
5741 for (const auto &MDOperand : MDScopeList->operands())
5742 if (auto *MDScope = dyn_cast<MDNode>(MDOperand))
5743 Container.insert(MDScope);
5744 };
5745
5746 Track(I->getMetadata(LLVMContext::MD_alias_scope), UsedAliasScopesAndLists);
5747 Track(I->getMetadata(LLVMContext::MD_noalias), UsedNoAliasScopesAndLists);
5748 }
5749
5752 if (!Decl)
5753 return false;
5754
5755 assert(Decl->use_empty() &&
5756 "llvm.experimental.noalias.scope.decl in use ?");
5757 const MDNode *MDSL = Decl->getScopeList();
5758 assert(MDSL->getNumOperands() == 1 &&
5759 "llvm.experimental.noalias.scope should refer to a single scope");
5760 auto &MDOperand = MDSL->getOperand(0);
5761 if (auto *MD = dyn_cast<MDNode>(MDOperand))
5762 return !UsedAliasScopesAndLists.contains(MD) ||
5763 !UsedNoAliasScopesAndLists.contains(MD);
5764
5765 // Not an MDNode ? throw away.
5766 return true;
5767 }
5768};
5769
5770/// Populate the IC worklist from a function, by walking it in reverse
5771/// post-order and adding all reachable code to the worklist.
5772///
5773/// This has a couple of tricks to make the code faster and more powerful. In
5774/// particular, we constant fold and DCE instructions as we go, to avoid adding
5775/// them to the worklist (this significantly speeds up instcombine on code where
5776/// many instructions are dead or constant). Additionally, if we find a branch
5777/// whose condition is a known constant, we only visit the reachable successors.
5779 bool MadeIRChange = false;
5781 SmallVector<Instruction *, 128> InstrsForInstructionWorklist;
5782 DenseMap<Constant *, Constant *> FoldedConstants;
5783 AliasScopeTracker SeenAliasScopes;
5784
5785 auto HandleOnlyLiveSuccessor = [&](BasicBlock *BB, BasicBlock *LiveSucc) {
5786 for (BasicBlock *Succ : successors(BB))
5787 if (Succ != LiveSucc && DeadEdges.insert({BB, Succ}).second)
5788 for (PHINode &PN : Succ->phis())
5789 for (Use &U : PN.incoming_values())
5790 if (PN.getIncomingBlock(U) == BB && !isa<PoisonValue>(U)) {
5791 U.set(PoisonValue::get(PN.getType()));
5792 MadeIRChange = true;
5793 }
5794 };
5795
5796 for (BasicBlock *BB : RPOT) {
5797 if (!BB->isEntryBlock() && all_of(predecessors(BB), [&](BasicBlock *Pred) {
5798 return DeadEdges.contains({Pred, BB}) || DT.dominates(BB, Pred);
5799 })) {
5800 HandleOnlyLiveSuccessor(BB, nullptr);
5801 continue;
5802 }
5803 LiveBlocks.insert(BB);
5804
5805 for (Instruction &Inst : llvm::make_early_inc_range(*BB)) {
5806 // ConstantProp instruction if trivially constant.
5807 if (!Inst.use_empty() &&
5808 (Inst.getNumOperands() == 0 || isa<Constant>(Inst.getOperand(0))))
5809 if (Constant *C = ConstantFoldInstruction(&Inst, DL, &TLI)) {
5810 LLVM_DEBUG(dbgs() << "IC: ConstFold to: " << *C << " from: " << Inst
5811 << '\n');
5812 Inst.replaceAllUsesWith(C);
5813 ++NumConstProp;
5814 if (isInstructionTriviallyDead(&Inst, &TLI))
5815 Inst.eraseFromParent();
5816 MadeIRChange = true;
5817 continue;
5818 }
5819
5820 // See if we can constant fold its operands.
5821 for (Use &U : Inst.operands()) {
5823 continue;
5824
5825 auto *C = cast<Constant>(U);
5826 Constant *&FoldRes = FoldedConstants[C];
5827 if (!FoldRes)
5828 FoldRes = ConstantFoldConstant(C, DL, &TLI);
5829
5830 if (FoldRes != C) {
5831 LLVM_DEBUG(dbgs() << "IC: ConstFold operand of: " << Inst
5832 << "\n Old = " << *C
5833 << "\n New = " << *FoldRes << '\n');
5834 U = FoldRes;
5835 MadeIRChange = true;
5836 }
5837 }
5838
5839 // Skip processing debug and pseudo intrinsics in InstCombine. Processing
5840 // these call instructions consumes non-trivial amount of time and
5841 // provides no value for the optimization.
5842 if (!Inst.isDebugOrPseudoInst()) {
5843 InstrsForInstructionWorklist.push_back(&Inst);
5844 SeenAliasScopes.analyse(&Inst);
5845 }
5846 }
5847
5848 // If this is a branch or switch on a constant, mark only the single
5849 // live successor. Otherwise assume all successors are live.
5850 Instruction *TI = BB->getTerminator();
5851 if (BranchInst *BI = dyn_cast<BranchInst>(TI); BI && BI->isConditional()) {
5852 if (isa<UndefValue>(BI->getCondition())) {
5853 // Branch on undef is UB.
5854 HandleOnlyLiveSuccessor(BB, nullptr);
5855 continue;
5856 }
5857 if (auto *Cond = dyn_cast<ConstantInt>(BI->getCondition())) {
5858 bool CondVal = Cond->getZExtValue();
5859 HandleOnlyLiveSuccessor(BB, BI->getSuccessor(!CondVal));
5860 continue;
5861 }
5862 } else if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
5863 if (isa<UndefValue>(SI->getCondition())) {
5864 // Switch on undef is UB.
5865 HandleOnlyLiveSuccessor(BB, nullptr);
5866 continue;
5867 }
5868 if (auto *Cond = dyn_cast<ConstantInt>(SI->getCondition())) {
5869 HandleOnlyLiveSuccessor(BB,
5870 SI->findCaseValue(Cond)->getCaseSuccessor());
5871 continue;
5872 }
5873 }
5874 }
5875
5876 // Remove instructions inside unreachable blocks. This prevents the
5877 // instcombine code from having to deal with some bad special cases, and
5878 // reduces use counts of instructions.
5879 for (BasicBlock &BB : F) {
5880 if (LiveBlocks.count(&BB))
5881 continue;
5882
5883 unsigned NumDeadInstInBB;
5884 NumDeadInstInBB = removeAllNonTerminatorAndEHPadInstructions(&BB);
5885
5886 MadeIRChange |= NumDeadInstInBB != 0;
5887 NumDeadInst += NumDeadInstInBB;
5888 }
5889
5890 // Once we've found all of the instructions to add to instcombine's worklist,
5891 // add them in reverse order. This way instcombine will visit from the top
5892 // of the function down. This jives well with the way that it adds all uses
5893 // of instructions to the worklist after doing a transformation, thus avoiding
5894 // some N^2 behavior in pathological cases.
5895 Worklist.reserve(InstrsForInstructionWorklist.size());
5896 for (Instruction *Inst : reverse(InstrsForInstructionWorklist)) {
5897 // DCE instruction if trivially dead. As we iterate in reverse program
5898 // order here, we will clean up whole chains of dead instructions.
5899 if (isInstructionTriviallyDead(Inst, &TLI) ||
5900 SeenAliasScopes.isNoAliasScopeDeclDead(Inst)) {
5901 ++NumDeadInst;
5902 LLVM_DEBUG(dbgs() << "IC: DCE: " << *Inst << '\n');
5903 salvageDebugInfo(*Inst);
5904 Inst->eraseFromParent();
5905 MadeIRChange = true;
5906 continue;
5907 }
5908
5909 Worklist.push(Inst);
5910 }
5911
5912 return MadeIRChange;
5913}
5914
5916 // Collect backedges.
5918 for (BasicBlock *BB : RPOT) {
5919 Visited.insert(BB);
5920 for (BasicBlock *Succ : successors(BB))
5921 if (Visited.contains(Succ))
5922 BackEdges.insert({BB, Succ});
5923 }
5924 ComputedBackEdges = true;
5925}
5926
5932 const InstCombineOptions &Opts) {
5933 auto &DL = F.getDataLayout();
5934 bool VerifyFixpoint = Opts.VerifyFixpoint &&
5935 !F.hasFnAttribute("instcombine-no-verify-fixpoint");
5936
5937 /// Builder - This is an IRBuilder that automatically inserts new
5938 /// instructions into the worklist when they are created.
5940 F.getContext(), TargetFolder(DL),
5941 IRBuilderCallbackInserter([&Worklist, &AC](Instruction *I) {
5942 Worklist.add(I);
5943 if (auto *Assume = dyn_cast<AssumeInst>(I))
5944 AC.registerAssumption(Assume);
5945 }));
5946
5948
5949 // Lower dbg.declare intrinsics otherwise their value may be clobbered
5950 // by instcombiner.
5951 bool MadeIRChange = false;
5953 MadeIRChange = LowerDbgDeclare(F);
5954
5955 // Iterate while there is work to do.
5956 unsigned Iteration = 0;
5957 while (true) {
5958 if (Iteration >= Opts.MaxIterations && !VerifyFixpoint) {
5959 LLVM_DEBUG(dbgs() << "\n\n[IC] Iteration limit #" << Opts.MaxIterations
5960 << " on " << F.getName()
5961 << " reached; stopping without verifying fixpoint\n");
5962 break;
5963 }
5964
5965 ++Iteration;
5966 ++NumWorklistIterations;
5967 LLVM_DEBUG(dbgs() << "\n\nINSTCOMBINE ITERATION #" << Iteration << " on "
5968 << F.getName() << "\n");
5969
5970 InstCombinerImpl IC(Worklist, Builder, F, AA, AC, TLI, TTI, DT, ORE, BFI,
5971 BPI, PSI, DL, RPOT);
5973 bool MadeChangeInThisIteration = IC.prepareWorklist(F);
5974 MadeChangeInThisIteration |= IC.run();
5975 if (!MadeChangeInThisIteration)
5976 break;
5977
5978 MadeIRChange = true;
5979 if (Iteration > Opts.MaxIterations) {
5981 "Instruction Combining on " + Twine(F.getName()) +
5982 " did not reach a fixpoint after " + Twine(Opts.MaxIterations) +
5983 " iterations. " +
5984 "Use 'instcombine<no-verify-fixpoint>' or function attribute "
5985 "'instcombine-no-verify-fixpoint' to suppress this error.");
5986 }
5987 }
5988
5989 if (Iteration == 1)
5990 ++NumOneIteration;
5991 else if (Iteration == 2)
5992 ++NumTwoIterations;
5993 else if (Iteration == 3)
5994 ++NumThreeIterations;
5995 else
5996 ++NumFourOrMoreIterations;
5997
5998 return MadeIRChange;
5999}
6000
6002
6004 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
6005 static_cast<PassInfoMixin<InstCombinePass> *>(this)->printPipeline(
6006 OS, MapClassName2PassName);
6007 OS << '<';
6008 OS << "max-iterations=" << Options.MaxIterations << ";";
6009 OS << (Options.VerifyFixpoint ? "" : "no-") << "verify-fixpoint";
6010 OS << '>';
6011}
6012
6013char InstCombinePass::ID = 0;
6014
6017 auto &LRT = AM.getResult<LastRunTrackingAnalysis>(F);
6018 // No changes since last InstCombine pass, exit early.
6019 if (LRT.shouldSkip(&ID))
6020 return PreservedAnalyses::all();
6021
6022 auto &AC = AM.getResult<AssumptionAnalysis>(F);
6023 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
6024 auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
6026 auto &TTI = AM.getResult<TargetIRAnalysis>(F);
6027
6028 auto *AA = &AM.getResult<AAManager>(F);
6029 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
6030 ProfileSummaryInfo *PSI =
6031 MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
6032 auto *BFI = (PSI && PSI->hasProfileSummary()) ?
6033 &AM.getResult<BlockFrequencyAnalysis>(F) : nullptr;
6035
6036 if (!combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE,
6037 BFI, BPI, PSI, Options)) {
6038 // No changes, all analyses are preserved.
6039 LRT.update(&ID, /*Changed=*/false);
6040 return PreservedAnalyses::all();
6041 }
6042
6043 // Mark all the analyses that instcombine updates as preserved.
6045 LRT.update(&ID, /*Changed=*/true);
6048 return PA;
6049}
6050
6066
6068 if (skipFunction(F))
6069 return false;
6070
6071 // Required analyses.
6072 auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
6073 auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
6074 auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
6076 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
6078
6079 // Optional analyses.
6080 ProfileSummaryInfo *PSI =
6082 BlockFrequencyInfo *BFI =
6083 (PSI && PSI->hasProfileSummary()) ?
6085 nullptr;
6086 BranchProbabilityInfo *BPI = nullptr;
6087 if (auto *WrapperPass =
6089 BPI = &WrapperPass->getBPI();
6090
6091 return combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE,
6092 BFI, BPI, PSI, InstCombineOptions());
6093}
6094
6096
6100
6102 "Combine redundant instructions", false, false)
6113 "Combine redundant instructions", false, false)
6114
6115// Initialization Routines
6119
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
Rewrite undef for PHI
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This is the interface for LLVM's primary stateless and local alias analysis.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool willNotOverflow(BinaryOpIntrinsic *BO, LazyValueInfo *LVI)
DXIL Resource Access
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
This file defines the DenseMap class.
static bool isSigned(unsigned int Opcode)
This is the interface for a simple mod/ref and alias analysis over globals.
Hexagon Common GEP
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This header defines various interfaces for pass management in LLVM.
This defines the Use class.
iv Induction Variable Users
Definition IVUsers.cpp:48
static bool leftDistributesOverRight(Instruction::BinaryOps LOp, bool HasNUW, bool HasNSW, Intrinsic::ID ROp)
Return whether "X LOp (Y ROp Z)" is always equal to "(X LOp Y) ROp (X LOp Z)".
This file provides internal interfaces used to implement the InstCombine.
This file provides the primary interface to the instcombine pass.
static Value * simplifySwitchOnSelectUsingRanges(SwitchInst &SI, SelectInst *Select, bool IsTrueArm)
static bool isUsedWithinShuffleVector(Value *V)
static bool isNeverEqualToUnescapedAlloc(Value *V, const TargetLibraryInfo &TLI, Instruction *AI)
static bool shorter_filter(const Value *LHS, const Value *RHS)
static Instruction * combineConstantOffsets(GetElementPtrInst &GEP, InstCombinerImpl &IC)
Combine constant offsets separated by variable offsets.
static Instruction * foldSelectGEP(GetElementPtrInst &GEP, InstCombiner::BuilderTy &Builder)
Thread a GEP operation with constant indices through the constant true/false arms of a select.
static bool shouldMergeGEPs(GEPOperator &GEP, GEPOperator &Src)
static cl::opt< unsigned > MaxArraySize("instcombine-maxarray-size", cl::init(1024), cl::desc("Maximum array size considered when doing a combine"))
static cl::opt< unsigned > ShouldLowerDbgDeclare("instcombine-lower-dbg-declare", cl::Hidden, cl::init(true))
static bool hasNoSignedWrap(BinaryOperator &I)
static bool simplifyAssocCastAssoc(BinaryOperator *BinOp1, InstCombinerImpl &IC)
Combine constant operands of associative operations either before or after a cast to eliminate one of...
static bool combineInstructionsOverFunction(Function &F, InstructionWorklist &Worklist, AliasAnalysis *AA, AssumptionCache &AC, TargetLibraryInfo &TLI, TargetTransformInfo &TTI, DominatorTree &DT, OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI, BranchProbabilityInfo *BPI, ProfileSummaryInfo *PSI, const InstCombineOptions &Opts)
static Value * simplifyInstructionWithPHI(Instruction &I, PHINode *PN, Value *InValue, BasicBlock *InBB, const DataLayout &DL, const SimplifyQuery SQ)
static bool shouldCanonicalizeGEPToPtrAdd(GetElementPtrInst &GEP)
Return true if we should canonicalize the gep to an i8 ptradd.
static void ClearSubclassDataAfterReassociation(BinaryOperator &I)
Conservatively clears subclassOptionalData after a reassociation or commutation.
static Value * getIdentityValue(Instruction::BinaryOps Opcode, Value *V)
This function returns identity value for given opcode, which can be used to factor patterns like (X *...
static Value * foldFrexpOfSelect(ExtractValueInst &EV, IntrinsicInst *FrexpCall, SelectInst *SelectInst, InstCombiner::BuilderTy &Builder)
static std::optional< std::pair< Value *, Value * > > matchSymmetricPhiNodesPair(PHINode *LHS, PHINode *RHS)
static Value * foldOperationIntoSelectOperand(Instruction &I, SelectInst *SI, Value *NewOp, InstCombiner &IC)
static Instruction * canonicalizeGEPOfConstGEPI8(GetElementPtrInst &GEP, GEPOperator *Src, InstCombinerImpl &IC)
static Instruction * tryToMoveFreeBeforeNullTest(CallInst &FI, const DataLayout &DL)
Move the call to free before a NULL test.
static Value * simplifyOperationIntoSelectOperand(Instruction &I, SelectInst *SI, bool IsTrueArm)
static bool rightDistributesOverLeft(Instruction::BinaryOps LOp, Instruction::BinaryOps ROp)
Return whether "(X LOp Y) ROp Z" is always equal to "(X ROp Z) LOp (Y ROp Z)".
static Value * tryFactorization(BinaryOperator &I, const SimplifyQuery &SQ, InstCombiner::BuilderTy &Builder, Instruction::BinaryOps InnerOpcode, Value *A, Value *B, Value *C, Value *D)
This tries to simplify binary operations by factorizing out common terms (e.
static bool isRemovableWrite(CallBase &CB, Value *UsedV, const TargetLibraryInfo &TLI)
Given a call CB which uses an address UsedV, return true if we can prove the call's only possible eff...
static Instruction::BinaryOps getBinOpsForFactorization(Instruction::BinaryOps TopOpcode, BinaryOperator *Op, Value *&LHS, Value *&RHS, BinaryOperator *OtherOp)
This function predicates factorization using distributive laws.
static bool hasNoUnsignedWrap(BinaryOperator &I)
static bool SoleWriteToDeadLocal(Instruction *I, TargetLibraryInfo &TLI)
Check for case where the call writes to an otherwise dead alloca.
static cl::opt< unsigned > MaxSinkNumUsers("instcombine-max-sink-users", cl::init(32), cl::desc("Maximum number of undroppable users for instruction sinking"))
static Instruction * foldGEPOfPhi(GetElementPtrInst &GEP, PHINode *PN, IRBuilderBase &Builder)
static std::optional< ModRefInfo > isAllocSiteRemovable(Instruction *AI, SmallVectorImpl< WeakTrackingVH > &Users, const TargetLibraryInfo &TLI, bool KnowInit)
static bool isCatchAll(EHPersonality Personality, Constant *TypeInfo)
Return 'true' if the given typeinfo will match anything.
static cl::opt< bool > EnableCodeSinking("instcombine-code-sinking", cl::desc("Enable code sinking"), cl::init(true))
static bool maintainNoSignedWrap(BinaryOperator &I, Value *B, Value *C)
static GEPNoWrapFlags getMergedGEPNoWrapFlags(GEPOperator &GEP1, GEPOperator &GEP2)
Determine nowrap flags for (gep (gep p, x), y) to (gep p, (x + y)) transform.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
This file contains the declarations for metadata subclasses.
#define T
uint64_t IntrinsicInst * II
static bool IsSelect(MachineInstr &MI)
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
const SmallVectorImpl< MachineOperand > & Cond
static unsigned getNumElements(Type *Ty)
unsigned OpIndex
BaseType
A given derived pointer can have multiple base pointers through phi/selects.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:114
static unsigned getScalarSizeInBits(Type *Ty)
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
This pass exposes codegen information to IR-level passes.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition VPlanSLP.cpp:247
Value * RHS
Value * LHS
static const uint32_t IV[8]
Definition blake3_impl.h:83
bool isNoAliasScopeDeclDead(Instruction *Inst)
void analyse(Instruction *I)
A manager for alias analyses.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:344
static LLVM_ABI unsigned int semanticsPrecision(const fltSemantics &)
Definition APFloat.cpp:296
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:234
static LLVM_ABI void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
Definition APInt.cpp:1758
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition APInt.h:423
static LLVM_ABI void sdivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Definition APInt.cpp:1890
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:936
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:371
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:380
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1488
LLVM_ABI APInt sadd_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1928
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition APInt.h:827
LLVM_ABI APInt smul_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1960
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition APInt.h:334
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition APInt.h:1150
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:440
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:306
LLVM_ABI APInt ssub_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1941
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition APInt.h:851
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition Pass.cpp:270
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
Definition ArrayRef.h:224
size_t size() const
size - Get the array size.
Definition ArrayRef.h:147
Class to represent array types.
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
uint64_t getNumElements() const
Type * getElementType() const
A function analysis which provides an AssumptionCache.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
LLVM_ABI void registerAssumption(AssumeInst *CI)
Add an @llvm.assume intrinsic to this function's cache.
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:69
LLVM_ABI uint64_t getDereferenceableBytes() const
Returns the number of dereferenceable bytes from the dereferenceable attribute.
bool isValid() const
Return true if the attribute is any kind of attribute.
Definition Attributes.h:223
Legacy wrapper pass to provide the BasicAAResult object.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition BasicBlock.h:528
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
LLVM_ABI iterator_range< filter_iterator< BasicBlock::const_iterator, std::function< bool(const Instruction &)> > > instructionsWithoutDebug(bool SkipPseudoOp=true) const
Return a const iterator range over the instructions in the block, skipping any debug instructions.
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
LLVM_ABI bool isEntryBlock() const
Return true if this is the entry block of the containing function.
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction & front() const
Definition BasicBlock.h:482
LLVM_ABI const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
LLVM_ABI const_iterator getFirstNonPHIOrDbgOrAlloca() const
Returns an iterator to the first instruction in this block that is not a PHINode, a debug intrinsic,...
size_t size() const
Definition BasicBlock.h:480
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition BasicBlock.h:233
static LLVM_ABI BinaryOperator * CreateNeg(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Helper functions to construct and inspect unary operations (NEG and NOT) via binary operators SUB and...
BinaryOps getOpcode() const
Definition InstrTypes.h:374
static LLVM_ABI BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
static BinaryOperator * CreateNUW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name="")
Definition InstrTypes.h:294
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Conditional or Unconditional Branch instruction.
LLVM_ABI void swapSuccessors()
Swap the successors of this branch instruction.
bool isConditional() const
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
Analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
void setAttributes(AttributeList A)
Set the attributes for this call.
bool doesNotThrow() const
Determine if the call cannot unwind.
Value * getArgOperand(unsigned i) const
AttributeList getAttributes() const
Return the attributes for this call.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:676
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:699
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:701
@ ICMP_NE
not equal
Definition InstrTypes.h:698
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:827
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:789
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
ConstantArray - Constant Array Declarations.
Definition Constants.h:433
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
A vector constant whose element type is a simple 1/2/4/8-byte integer or float/double,...
Definition Constants.h:776
static LLVM_ABI Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getNot(Constant *C)
static LLVM_ABI Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getBinOpIdentity(unsigned Opcode, Type *Ty, bool AllowRHSConstant=false, bool NSZ=false)
Return the identity constant for a binary opcode.
static LLVM_ABI Constant * getNeg(Constant *C, bool HasNSW=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
static LLVM_ABI ConstantInt * getBool(LLVMContext &Context, bool V)
This class represents a range of values.
LLVM_ABI bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const
Set up Pred and RHS such that ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.
static LLVM_ABI ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
LLVM_ABI bool contains(const APInt &Val) const
Return true if the specified value is in the set.
static LLVM_ABI ConstantRange makeExactNoWrapRegion(Instruction::BinaryOps BinOp, const APInt &Other, unsigned NoWrapKind)
Produce the range that contains X if and only if "X BinOp Other" does not wrap.
Constant Vector Declarations.
Definition Constants.h:517
static LLVM_ABI Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
Definition Constant.h:43
static LLVM_ABI Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
static LLVM_ABI Constant * replaceUndefsWith(Constant *C, Constant *Replacement)
Try to replace undefined constant C or undefined elements in C with Replacement.
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
const Constant * stripPointerCasts() const
Definition Constant.h:219
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
LLVM_ABI Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
LLVM_ABI bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition Constants.cpp:90
static LLVM_ABI DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:63
Record of a variable value-assignment, aka a non instruction representation of the dbg....
static bool shouldExecute(unsigned CounterName)
Identifies a unique instance of a variable.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition DenseMap.h:194
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:167
bool empty() const
Definition DenseMap.h:109
iterator end()
Definition DenseMap.h:81
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:222
Analysis pass which computes a DominatorTree.
Definition Dominators.h:284
Legacy analysis pass which computes a DominatorTree.
Definition Dominators.h:322
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:165
This instruction extracts a struct member or array element value from an aggregate value.
ArrayRef< unsigned > getIndices() const
iterator_range< idx_iterator > indices() const
idx_iterator idx_end() const
static ExtractValueInst * Create(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
idx_iterator idx_begin() const
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition Operator.h:200
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:22
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:314
FunctionPass(char &pid)
Definition Pass.h:316
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition Pass.cpp:188
const BasicBlock & getEntryBlock() const
Definition Function.h:807
Represents flags for the getelementptr instruction/expression.
static GEPNoWrapFlags inBounds()
static GEPNoWrapFlags all()
static GEPNoWrapFlags noUnsignedWrap()
GEPNoWrapFlags intersectForReassociate(GEPNoWrapFlags Other) const
Given (gep (gep p, x), y), determine the nowrap flags for (gep (gep, p, y), x).
bool hasNoUnsignedWrap() const
bool isInBounds() const
GEPNoWrapFlags intersectForOffsetAdd(GEPNoWrapFlags Other) const
Given (gep (gep p, x), y), determine the nowrap flags for (gep p, x+y).
static GEPNoWrapFlags none()
GEPNoWrapFlags getNoWrapFlags() const
Definition Operator.h:425
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static LLVM_ABI Type * getTypeAtIndex(Type *Ty, Value *Idx)
Return the type of the element at the given index of an indexable type.
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
static LLVM_ABI Type * getIndexedType(Type *Ty, ArrayRef< Value * > IdxList)
Returns the result type of a getelementptr with the given source element type and indexes.
static GetElementPtrInst * CreateInBounds(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Create an "inbounds" getelementptr.
Legacy wrapper pass to provide the GlobalsAAResult object.
This instruction compares its operands according to the predicate given to the constructor.
CmpPredicate getCmpPredicate() const
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
Common base class shared among various IRBuilders.
Definition IRBuilder.h:114
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Definition IRBuilder.h:2039
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
Definition IRBuilder.h:538
Provides an 'InsertHelper' that calls a user-provided callback after performing the default insertion...
Definition IRBuilder.h:75
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2788
This instruction inserts a struct field of array element value into an aggregate value.
static InsertValueInst * Create(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
LLVM_ABI InstCombinePass(InstCombineOptions Opts={})
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Instruction * foldBinOpOfSelectAndCastOfSelectCondition(BinaryOperator &I)
Tries to simplify binops of select and cast of the select condition.
Instruction * foldBinOpIntoSelectOrPhi(BinaryOperator &I)
This is a convenience wrapper function for the above two functions.
bool SimplifyAssociativeOrCommutative(BinaryOperator &I)
Performs a few simplifications for operators which are associative or commutative.
Instruction * visitGEPOfGEP(GetElementPtrInst &GEP, GEPOperator *Src)
Value * foldUsingDistributiveLaws(BinaryOperator &I)
Tries to simplify binary operations which some other binary operation distributes over.
Instruction * foldBinOpShiftWithShift(BinaryOperator &I)
Instruction * visitUnreachableInst(UnreachableInst &I)
Instruction * foldOpIntoPhi(Instruction &I, PHINode *PN, bool AllowMultipleUses=false)
Given a binary operator, cast instruction, or select which has a PHI node as operand #0,...
void handleUnreachableFrom(Instruction *I, SmallVectorImpl< BasicBlock * > &Worklist)
Value * SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &PoisonElts, unsigned Depth=0, bool AllowMultipleUsers=false) override
The specified value produces a vector with any number of elements.
Instruction * visitFreeze(FreezeInst &I)
void handlePotentiallyDeadBlocks(SmallVectorImpl< BasicBlock * > &Worklist)
bool prepareWorklist(Function &F)
Perform early cleanup and prepare the InstCombine worklist.
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false, bool SimplifyBothArms=false)
Given an instruction with a select as one operand and a constant as the other operand,...
Instruction * visitFree(CallInst &FI, Value *FreedOp)
Instruction * visitExtractValueInst(ExtractValueInst &EV)
void handlePotentiallyDeadSuccessors(BasicBlock *BB, BasicBlock *LiveSucc)
Instruction * visitUnconditionalBranchInst(BranchInst &BI)
Instruction * foldBinopWithRecurrence(BinaryOperator &BO)
Try to fold binary operators whose operands are simple interleaved recurrences to a single recurrence...
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Instruction * visitLandingPadInst(LandingPadInst &LI)
Instruction * visitReturnInst(ReturnInst &RI)
Instruction * visitSwitchInst(SwitchInst &SI)
Instruction * foldBinopWithPhiOperands(BinaryOperator &BO)
For a binary operator with 2 phi operands, try to hoist the binary operation before the phi.
bool mergeStoreIntoSuccessor(StoreInst &SI)
Try to transform: if () { *P = v1; } else { *P = v2 } or: *P = v1; if () { *P = v2; }...
Instruction * tryFoldInstWithCtpopWithNot(Instruction *I)
void CreateNonTerminatorUnreachable(Instruction *InsertAt)
Create and insert the idiom we use to indicate a block is unreachable without having to rewrite the C...
Value * pushFreezeToPreventPoisonFromPropagating(FreezeInst &FI)
bool run()
Run the combiner over the entire worklist until it is empty.
Instruction * foldVectorBinop(BinaryOperator &Inst)
Canonicalize the position of binops relative to shufflevector.
bool removeInstructionsBeforeUnreachable(Instruction &I)
Value * SimplifySelectsFeedingBinaryOp(BinaryOperator &I, Value *LHS, Value *RHS)
void tryToSinkInstructionDbgVariableRecords(Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock, BasicBlock *DestBlock, SmallVectorImpl< DbgVariableRecord * > &DPUsers)
void addDeadEdge(BasicBlock *From, BasicBlock *To, SmallVectorImpl< BasicBlock * > &Worklist)
Constant * unshuffleConstant(ArrayRef< int > ShMask, Constant *C, VectorType *NewCTy)
Find a constant NewC that has property: shuffle(NewC, ShMask) = C Returns nullptr if such a constant ...
Instruction * visitAllocSite(Instruction &FI)
Instruction * visitGetElementPtrInst(GetElementPtrInst &GEP)
Instruction * visitBranchInst(BranchInst &BI)
Value * tryFactorizationFolds(BinaryOperator &I)
This tries to simplify binary operations by factorizing out common terms (e.
Instruction * foldFreezeIntoRecurrence(FreezeInst &I, PHINode *PN)
Value * SimplifyDemandedUseFPClass(Value *V, FPClassTest DemandedMask, KnownFPClass &Known, Instruction *CxtI, unsigned Depth=0)
Attempts to replace V with a simpler value based on the demanded floating-point classes.
bool tryToSinkInstruction(Instruction *I, BasicBlock *DestBlock)
Try to move the specified instruction from its current block into the beginning of DestBlock,...
bool freezeOtherUses(FreezeInst &FI)
void freelyInvertAllUsersOf(Value *V, Value *IgnoredUser=nullptr)
Freely adapt every user of V as-if V was changed to !V.
The core instruction combiner logic.
SimplifyQuery SQ
const DataLayout & getDataLayout() const
IRBuilder< TargetFolder, IRBuilderCallbackInserter > BuilderTy
An IRBuilder that automatically inserts new instructions into the worklist.
bool isFreeToInvert(Value *V, bool WillInvertAllUses, bool &DoesConsume)
Return true if the specified value is free to invert (apply ~ to).
static unsigned getComplexity(Value *V)
Assign a complexity or rank value to LLVM Values.
TargetLibraryInfo & TLI
unsigned ComputeNumSignBits(const Value *Op, const Instruction *CxtI=nullptr, unsigned Depth=0) const
Instruction * InsertNewInstBefore(Instruction *New, BasicBlock::iterator Old)
Inserts an instruction New before instruction Old.
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
uint64_t MaxArraySizeForCombine
Maximum size of array considered when transforming.
static bool shouldAvoidAbsorbingNotIntoSelect(const SelectInst &SI)
void replaceUse(Use &U, Value *NewValue)
Replace use and add the previously used value to the worklist.
static bool isCanonicalPredicate(CmpPredicate Pred)
Predicate canonicalization reduces the number of patterns that need to be matched by other transforms...
InstructionWorklist & Worklist
A worklist of the instructions that need to be simplified.
Instruction * InsertNewInstWith(Instruction *New, BasicBlock::iterator Old)
Same as InsertNewInstBefore, but also sets the debug loc.
BranchProbabilityInfo * BPI
ReversePostOrderTraversal< BasicBlock * > & RPOT
const DataLayout & DL
DomConditionCache DC
const bool MinimizeSize
void computeKnownBits(const Value *V, KnownBits &Known, const Instruction *CxtI, unsigned Depth=0) const
std::optional< Instruction * > targetInstCombineIntrinsic(IntrinsicInst &II)
AssumptionCache & AC
void addToWorklist(Instruction *I)
Value * getFreelyInvertedImpl(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume, unsigned Depth)
Return nonnull value if V is free to invert under the condition of WillInvertAllUses.
SmallDenseSet< std::pair< const BasicBlock *, const BasicBlock * >, 8 > BackEdges
Backedges, used to avoid pushing instructions across backedges in cases where this may result in infi...
std::optional< Value * > targetSimplifyDemandedVectorEltsIntrinsic(IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3, std::function< void(Instruction *, unsigned, APInt, APInt &)> SimplifyAndSetOp)
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
DominatorTree & DT
static Constant * getSafeVectorConstantForBinop(BinaryOperator::BinaryOps Opcode, Constant *In, bool IsRHSConstant)
Some binary operators require special handling to avoid poison and undefined behavior.
SmallDenseSet< std::pair< BasicBlock *, BasicBlock * >, 8 > DeadEdges
Edges that are known to never be taken.
std::optional< Value * > targetSimplifyDemandedUseBitsIntrinsic(IntrinsicInst &II, APInt DemandedMask, KnownBits &Known, bool &KnownBitsComputed)
BuilderTy & Builder
bool isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const
Value * getFreelyInverted(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume)
bool isBackEdge(const BasicBlock *From, const BasicBlock *To)
void visit(Iterator Start, Iterator End)
Definition InstVisitor.h:87
The legacy pass manager's instcombine pass.
Definition InstCombine.h:68
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool runOnFunction(Function &F) override
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
InstructionWorklist - This is the worklist management logic for InstCombine and other simplification ...
void add(Instruction *I)
Add instruction to the worklist.
LLVM_ABI void dropUBImplyingAttrsAndMetadata(ArrayRef< unsigned > Keep={})
Drop any attributes or metadata that can cause immediate undefined behavior.
static bool isBitwiseLogicOp(unsigned Opcode)
Determine if the Opcode is and/or/xor.
LLVM_ABI void copyIRFlags(const Value *V, bool IncludeWrapFlags=true)
Convenience method to copy supported exact, fast-math, and (optionally) wrapping flags from V to this...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void setAAMetadata(const AAMDNodes &N)
Sets the AA metadata on this instruction from the AAMDNodes structure.
LLVM_ABI bool isAssociative() const LLVM_READONLY
Return true if the instruction is associative:
LLVM_ABI bool isCommutative() const LLVM_READONLY
Return true if the instruction is commutative:
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
LLVM_ABI void setFastMathFlags(FastMathFlags FMF)
Convenience function for setting multiple fast-math flags on this instruction, which must be an opera...
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
bool isTerminator() const
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
LLVM_ABI bool willReturn() const LLVM_READONLY
Return true if the instruction will return (unwinding is considered as a form of returning control fl...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
bool isBitwiseLogicOp() const
Return true if this is and/or/xor.
bool isShift() const
LLVM_ABI void dropPoisonGeneratingFlags()
Drops flags that may cause this instruction to evaluate to poison despite having non-poison inputs.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
bool isIntDivRem() const
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:319
A wrapper class for inspecting calls to intrinsic functions.
Invoke instruction.
static InvokeInst * Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value * > Args, const Twine &NameStr, InsertPosition InsertBefore=nullptr)
The landingpad instruction holds all of the information necessary to generate correct exception handl...
bool isCleanup() const
Return 'true' if this landingpad instruction is a cleanup.
unsigned getNumClauses() const
Get the number of clauses for this landing pad.
static LLVM_ABI LandingPadInst * Create(Type *RetTy, unsigned NumReservedClauses, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedClauses is a hint for the number of incoming clauses that this landingpad w...
LLVM_ABI void addClause(Constant *ClauseVal)
Add a catch or filter clause to the landing pad.
bool isCatch(unsigned Idx) const
Return 'true' if the clause and index Idx is a catch clause.
bool isFilter(unsigned Idx) const
Return 'true' if the clause and index Idx is a filter clause.
Constant * getClause(unsigned Idx) const
Get the value of the clause at index Idx.
void setCleanup(bool V)
Indicate that this landingpad instruction is a cleanup.
A function/module analysis which provides an empty LastRunTrackingInfo.
This is an alternative analysis pass to BlockFrequencyInfoWrapperPass.
static void getLazyBFIAnalysisUsage(AnalysisUsage &AU)
Helper for client passes to set up the analysis usage on behalf of this pass.
An instruction for reading from memory.
Value * getPointerOperand()
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Metadata node.
Definition Metadata.h:1078
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1442
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1448
Tracking metadata reference owned by Metadata.
Definition Metadata.h:900
This is the common base class for memset/memcpy/memmove.
static LLVM_ABI MemoryLocation getForDest(const MemIntrinsic *MI)
Return a location representing the destination of a memory set or transfer.
Root of the metadata hierarchy.
Definition Metadata.h:64
Value * getLHS() const
Value * getRHS() const
static ICmpInst::Predicate getPredicate(Intrinsic::ID ID)
Returns the comparison predicate underlying the intrinsic.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
MDNode * getScopeList() const
OptimizationRemarkEmitter legacy analysis pass.
The optimization diagnostic interface.
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition Operator.h:78
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property.
Definition Operator.h:111
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property.
Definition Operator.h:105
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
static LLVM_ABI PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
AnalysisType * getAnalysisIfAvailable() const
getAnalysisIfAvailable<AnalysisType>() - Subclasses use this function to get analysis information tha...
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
Definition Constants.h:1468
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
Definition Analysis.h:151
PreservedAnalyses & preserve()
Mark an analysis as preserved.
Definition Analysis.h:132
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
bool hasProfileSummary() const
Returns true if profile summary is available.
A global registry used in conjunction with static constructors to make pluggable components (like tar...
Definition Registry.h:44
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, InsertPosition InsertBefore=nullptr)
This class represents the LLVM 'select' instruction.
const Value * getFalseValue() const
const Value * getCondition() const
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, const Instruction *MDFrom=nullptr)
const Value * getTrueValue() const
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:150
This instruction constructs a fixed permutation of two input vectors.
size_type size() const
Definition SmallPtrSet.h:99
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
Definition SetVector.h:338
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition SmallSet.h:133
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:183
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
typename SuperClass::iterator iterator
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
Multiway switch.
TargetFolder - Create constants with target dependent folding.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
bool has(LibFunc F) const
Tests whether a library function is available.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
Wrapper pass for TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:273
LLVM_ABI bool isScalableTy(SmallPtrSetImpl< const Type * > &Visited) const
Return true if this is a type whose size is a known multiple of vscale.
Definition Type.cpp:62
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:267
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:295
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:352
bool isStructTy() const
True if this is an instance of StructType.
Definition Type.h:261
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:198
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition Type.h:311
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:231
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:294
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:240
LLVM_ABI const fltSemantics & getFltSemantics() const
Definition Type.cpp:107
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
Use * op_iterator
Definition User.h:279
op_range operands()
Definition User.h:292
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition User.cpp:21
op_iterator op_begin()
Definition User.h:284
const Use & getOperandUse(unsigned i) const
Definition User.h:245
Value * getOperand(unsigned i) const
Definition User.h:232
unsigned getNumOperands() const
Definition User.h:254
op_iterator op_end()
Definition User.h:286
LLVM_ABI bool isDroppable() const
A droppable user is a user for which uses can be dropped without affecting correctness and should be ...
Definition User.cpp:115
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
const Value * stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, APInt &Offset) const
This is a wrapper around stripAndAccumulateConstantOffsets with the in-bounds requirement set to fals...
Definition Value.h:759
LLVM_ABI bool hasOneUser() const
Return true if there is exactly one user of this value.
Definition Value.cpp:166
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:439
iterator_range< user_iterator > users()
Definition Value.h:426
bool hasUseList() const
Check if this Value has a use-list.
Definition Value.h:344
LLVM_ABI bool hasNUses(unsigned N) const
Return true if this Value has exactly N uses.
Definition Value.cpp:150
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:701
bool use_empty() const
Definition Value.h:346
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.cpp:1099
LLVM_ABI uint64_t getPointerDereferenceableBytes(const DataLayout &DL, bool &CanBeNull, bool &CanBeFreed) const
Returns the number of bytes known to be dereferenceable for the pointer value.
Definition Value.cpp:881
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:396
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Value handle that is nullable, but tries to track the Value.
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition ilist_node.h:34
reverse_self_iterator getReverseIterator()
Definition ilist_node.h:126
self_iterator getIterator()
Definition ilist_node.h:123
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
A raw_ostream that writes to an std::string.
Changed
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Abstract Attribute helper functions.
Definition Attributor.h:165
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
cst_pred_ty< is_all_ones > m_AllOnes()
Match an integer or vector with all bits set.
class_match< PoisonValue > m_Poison()
Match an arbitrary poison constant.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
PtrAdd_match< PointerOpTy, OffsetOpTy > m_PtrAdd(const PointerOpTy &PointerOp, const OffsetOpTy &OffsetOp)
Matches GEP with i8 source element type.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
CmpClass_match< LHS, RHS, FCmpInst > m_FCmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
OneOps_match< OpTy, Instruction::Freeze > m_Freeze(const OpTy &Op)
Matches FreezeInst.
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
br_match m_UnconditionalBr(BasicBlock *&Succ)
ap_match< APInt > m_APIntAllowPoison(const APInt *&Res)
Match APInt while allowing poison in splat vector constants.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
BinOpPred_match< LHS, RHS, is_idiv_op > m_IDiv(const LHS &L, const RHS &R)
Matches integer division operations.
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
DisjointOr_match< LHS, RHS > m_DisjointOr(const LHS &L, const RHS &R)
constantexpr_match m_ConstantExpr()
Match a constant expression or a constant that contains a constant expression.
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
ap_match< APFloat > m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
cst_pred_ty< is_nonnegative > m_NonNegative()
Match an integer or vector of non-negative values.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
IntrinsicID_match m_Intrinsic()
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
ExtractValue_match< Ind, Val_t > m_ExtractValue(const Val_t &V)
Match a single index ExtractValue instruction.
match_combine_and< LTy, RTy > m_CombineAnd(const LTy &L, const RTy &R)
Combine two pattern matchers matching L && R.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
NNegZExt_match< OpTy > m_NNegZExt(const OpTy &Op)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
ThreeOps_match< decltype(m_Value()), LHS, RHS, Instruction::Select, true > m_c_Select(const LHS &L, const RHS &R)
Match Select(C, LHS, RHS) or Select(C, RHS, LHS)
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
BinaryOp_match< LHS, RHS, Instruction::UDiv > m_UDiv(const LHS &L, const RHS &R)
brc_match< Cond_t, bind_ty< BasicBlock >, bind_ty< BasicBlock > > m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F)
match_immconstant_ty m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
match_combine_or< BinaryOp_match< LHS, RHS, Instruction::Add >, DisjointOr_match< LHS, RHS > > m_AddLike(const LHS &L, const RHS &R)
Match either "add" or "or disjoint".
CastInst_match< OpTy, UIToFPInst > m_UIToFP(const OpTy &Op)
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
match_combine_or< CastInst_match< OpTy, SExtInst >, NNegZExt_match< OpTy > > m_SExtLike(const OpTy &Op)
Match either "sext" or "zext nneg".
BinaryOp_match< LHS, RHS, Instruction::SDiv > m_SDiv(const LHS &L, const RHS &R)
match_combine_or< OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoSignedWrap >, DisjointOr_match< LHS, RHS > > m_NSWAddLike(const LHS &L, const RHS &R)
Match either "add nsw" or "or disjoint".
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
AnyBinaryOp_match< LHS, RHS, true > m_c_BinOp(const LHS &L, const RHS &R)
Matches a BinaryOperator with LHS and RHS in either order.
CastInst_match< OpTy, SIToFPInst > m_SIToFP(const OpTy &Op)
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
BinOpPred_match< LHS, RHS, is_shift_op > m_Shift(const LHS &L, const RHS &R)
Matches shift operations.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
cstfp_pred_ty< is_non_zero_fp > m_NonZeroFP()
Match a floating-point non-zero.
m_Intrinsic_Ty< Opnd0 >::Ty m_VecReverse(const Opnd0 &Op0)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
match_combine_or< match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > >, match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > > > m_MaxOrMin(const LHS &L, const RHS &R)
auto m_Undef()
Match an arbitrary undef constant.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
match_combine_or< OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap >, DisjointOr_match< LHS, RHS > > m_NUWAddLike(const LHS &L, const RHS &R)
Match either "add nuw" or "or disjoint".
CastOperator_match< OpTy, Instruction::PtrToInt > m_PtrToInt(const OpTy &Op)
Matches PtrToInt.
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
initializer< Ty > init(const Ty &Val)
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
LLVM_ABI Intrinsic::ID getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID)
@ Offset
Definition DWP.cpp:477
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition STLExtras.h:829
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
void stable_sort(R &&Range)
Definition STLExtras.h:2058
LLVM_ABI void initializeInstructionCombiningPassPass(PassRegistry &)
LLVM_ABI unsigned removeAllNonTerminatorAndEHPadInstructions(BasicBlock *BB)
Remove all instructions from a basic block other than its terminator and any present EH pad instructi...
Definition Local.cpp:2485
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1725
LLVM_ABI Value * simplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef< Value * > Indices, GEPNoWrapFlags NW, const SimplifyQuery &Q)
Given operands for a GetElementPtrInst, fold the result or return null.
LLVM_ABI Constant * getInitialValueOfAllocation(const Value *V, const TargetLibraryInfo *TLI, Type *Ty)
If this is a call to an allocation function that initializes memory to a fixed value,...
bool succ_empty(const Instruction *I)
Definition CFG.h:257
LLVM_ABI Value * simplifyFreezeInst(Value *Op, const SimplifyQuery &Q)
Given an operand for a Freeze, see if we can fold the result.
LLVM_ABI FunctionPass * createInstructionCombiningPass()
LLVM_ABI void findDbgValues(Value *V, SmallVectorImpl< DbgVariableRecord * > &DbgVariableRecords)
Finds the dbg.values describing a value.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2472
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition Utils.cpp:1724
auto successors(const MachineBasicBlock *BB)
LLVM_ABI Constant * ConstantFoldInstruction(const Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldInstruction - Try to constant fold the specified instruction.
LLVM_ABI bool isRemovableAlloc(const CallBase *V, const TargetLibraryInfo *TLI)
Return true if this is a call to an allocation function that does not have side effects that we are r...
LLVM_ABI std::optional< StringRef > getAllocationFamily(const Value *I, const TargetLibraryInfo *TLI)
If a function is part of an allocation family (e.g.
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
LLVM_ABI Value * lowerObjectSizeCall(IntrinsicInst *ObjectSize, const DataLayout &DL, const TargetLibraryInfo *TLI, bool MustSucceed)
Try to turn a call to @llvm.objectsize into an integer value of the given Type.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
LLVM_ABI Value * simplifyInstructionWithOperands(Instruction *I, ArrayRef< Value * > NewOps, const SimplifyQuery &Q)
Like simplifyInstruction but the operands of I are replaced with NewOps.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2136
LLVM_ABI Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:632
gep_type_iterator gep_type_end(const User *GEP)
LLVM_ABI Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
LLVM_ABI Value * getReallocatedOperand(const CallBase *CB)
If this is a call to a realloc function, return the reallocated operand.
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
Definition APFloat.h:1537
LLVM_ABI bool isAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI)
Tests if a value is a call or invoke to a library function that allocates memory (either malloc,...
LLVM_ABI bool handleUnreachableTerminator(Instruction *I, SmallVectorImpl< Value * > &PoisonedValues)
If a terminator in an unreachable basic block has an operand of type Instruction, transform it into p...
Definition Local.cpp:2468
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:202
LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start, Value *&Step)
Attempt to match a simple first order recurrence cycle of the form: iv = phi Ty [Start,...
LLVM_ABI Value * simplifyAddInst(Value *LHS, Value *RHS, bool IsNSW, bool IsNUW, const SimplifyQuery &Q)
Given operands for an Add, fold the result or return null.
LLVM_ABI Constant * ConstantFoldConstant(const Constant *C, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldConstant - Fold the constant using the specified DataLayout.
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
constexpr bool has_single_bit(T Value) noexcept
Definition bit.h:147
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1732
LLVM_ABI bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
Definition Local.cpp:402
LLVM_ABI bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
LLVM_ABI Value * emitGEPOffset(IRBuilderBase *Builder, const DataLayout &DL, User *GEP, bool NoAssumptions=false)
Given a getelementptr instruction/constantexpr, emit the code necessary to compute the offset from th...
Definition Local.cpp:22
constexpr unsigned MaxAnalysisRecursionDepth
auto reverse(ContainerTy &&C)
Definition STLExtras.h:406
bool isModSet(const ModRefInfo MRI)
Definition ModRef.h:49
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI bool LowerDbgDeclare(Function &F)
Lowers dbg.declare records into appropriate set of dbg.value records.
Definition Local.cpp:1795
LLVM_ABI bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
generic_gep_type_iterator<> gep_type_iterator
LLVM_ABI void ConvertDebugDeclareToDebugValue(DbgVariableRecord *DVR, StoreInst *SI, DIBuilder &Builder)
Inserts a dbg.value record before a store to an alloca'd value that has an associated dbg....
Definition Local.cpp:1662
LLVM_ABI void salvageDebugInfoForDbgValues(Instruction &I, ArrayRef< DbgVariableRecord * > DPInsns)
Implementation of salvageDebugInfo, applying only to instructions in Insns, rather than all debug use...
Definition Local.cpp:2037
LLVM_ABI Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
LLVM_ABI bool canCreateUndefOrPoison(const Operator *Op, bool ConsiderFlagsAndMetadata=true)
canCreateUndefOrPoison returns true if Op can create undef or poison from non-undef & non-poison oper...
LLVM_ABI EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI Value * simplifyExtractValueInst(Value *Agg, ArrayRef< unsigned > Idxs, const SimplifyQuery &Q)
Given operands for an ExtractValueInst, fold the result or return null.
LLVM_ABI Constant * ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL)
Attempt to constant fold a binary operation with the specified operands.
LLVM_ABI bool replaceAllDbgUsesWith(Instruction &From, Value &To, Instruction &DomPoint, DominatorTree &DT)
Point debug users of From to To or salvage them.
Definition Local.cpp:2414
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
constexpr int PoisonMaskElem
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:323
ModRefInfo
Flags indicating whether a memory access modifies or references memory.
Definition ModRef.h:28
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
@ ModRef
The access may reference and may modify the value stored in memory.
Definition ModRef.h:36
@ Mod
The access may modify the value stored in memory.
Definition ModRef.h:34
@ NoModRef
The access neither references nor modifies the value stored in memory.
Definition ModRef.h:30
TargetTransformInfo TTI
LLVM_ABI Value * simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a BinaryOperator, fold the result or return null.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
DWARFExpression::Operation Op
bool isSafeToSpeculativelyExecuteWithVariableReplaced(const Instruction *I, bool IgnoreUBImplyingAttrs=true)
Don't use information from its non-constant operands.
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI Value * getFreedOperand(const CallBase *CB, const TargetLibraryInfo *TLI)
If this if a call to a free function, return the freed operand.
constexpr unsigned BitWidth
LLVM_ABI bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
LLVM_ABI Constant * getLosslessInvCast(Constant *C, Type *InvCastTo, unsigned CastOp, const DataLayout &DL, PreservedCastFlags *Flags=nullptr)
Try to cast C to InvC losslessly, satisfying CastOp(InvC) equals C, or CastOp(InvC) is a refined valu...
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition STLExtras.h:1961
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
gep_type_iterator gep_type_begin(const User *GEP)
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1897
cl::opt< bool > ProfcheckDisableMetadataFixes("profcheck-disable-metadata-fixes", cl::Hidden, cl::init(false), cl::desc("Disable metadata propagation fixes discovered through Issue #147390"))
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
bool equal(L &&LRange, R &&RRange)
Wrapper function around std::equal to detect if pair-wise elements between two ranges are the same.
Definition STLExtras.h:2088
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
AAResults AliasAnalysis
Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
LLVM_ABI void initializeInstCombine(PassRegistry &)
Initialize all passes linked into the InstCombine library.
LLVM_ABI void findDbgUsers(Value *V, SmallVectorImpl< DbgVariableRecord * > &DbgVariableRecords)
Finds the debug info records describing a value.
LLVM_ABI Constant * ConstantFoldBinaryInstruction(unsigned Opcode, Constant *V1, Constant *V2)
bool isRefSet(const ModRefInfo MRI)
Definition ModRef.h:52
LLVM_ABI std::optional< bool > isImpliedCondition(const Value *LHS, const Value *RHS, const DataLayout &DL, bool LHSIsTrue=true, unsigned Depth=0)
Return true if RHS is known to be implied true by LHS.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:180
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:869
#define N
unsigned countMinLeadingOnes() const
Returns the minimum number of leading one bits.
Definition KnownBits.h:251
unsigned getBitWidth() const
Get the bit width of this value.
Definition KnownBits.h:44
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:248
A CRTP mix-in to automatically provide informational APIs needed for passes.
Definition PassManager.h:70
SimplifyQuery getWithInstruction(const Instruction *I) const