LLVM 22.0.0git
InstructionCombining.cpp
Go to the documentation of this file.
1//===- InstructionCombining.cpp - Combine multiple instructions -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// InstructionCombining - Combine instructions to form fewer, simple
10// instructions. This pass does not modify the CFG. This pass is where
11// algebraic simplification happens.
12//
13// This pass combines things like:
14// %Y = add i32 %X, 1
15// %Z = add i32 %Y, 1
16// into:
17// %Z = add i32 %X, 2
18//
19// This is a simple worklist driven algorithm.
20//
21// This pass guarantees that the following canonicalizations are performed on
22// the program:
23// 1. If a binary operator has a constant operand, it is moved to the RHS
24// 2. Bitwise operators with constant operands are always grouped so that
25// shifts are performed first, then or's, then and's, then xor's.
26// 3. Compare instructions are converted from <,>,<=,>= to ==,!= if possible
27// 4. All cmp instructions on boolean values are replaced with logical ops
28// 5. add X, X is represented as (X*2) => (X << 1)
29// 6. Multiplies with a power-of-two constant argument are transformed into
30// shifts.
31// ... etc.
32//
33//===----------------------------------------------------------------------===//
34
35#include "InstCombineInternal.h"
36#include "llvm/ADT/APFloat.h"
37#include "llvm/ADT/APInt.h"
38#include "llvm/ADT/ArrayRef.h"
39#include "llvm/ADT/DenseMap.h"
42#include "llvm/ADT/Statistic.h"
47#include "llvm/Analysis/CFG.h"
62#include "llvm/IR/BasicBlock.h"
63#include "llvm/IR/CFG.h"
64#include "llvm/IR/Constant.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DIBuilder.h"
67#include "llvm/IR/DataLayout.h"
68#include "llvm/IR/DebugInfo.h"
70#include "llvm/IR/Dominators.h"
72#include "llvm/IR/Function.h"
74#include "llvm/IR/IRBuilder.h"
75#include "llvm/IR/InstrTypes.h"
76#include "llvm/IR/Instruction.h"
79#include "llvm/IR/Intrinsics.h"
80#include "llvm/IR/Metadata.h"
81#include "llvm/IR/Operator.h"
82#include "llvm/IR/PassManager.h"
84#include "llvm/IR/Type.h"
85#include "llvm/IR/Use.h"
86#include "llvm/IR/User.h"
87#include "llvm/IR/Value.h"
88#include "llvm/IR/ValueHandle.h"
93#include "llvm/Support/Debug.h"
102#include <algorithm>
103#include <cassert>
104#include <cstdint>
105#include <memory>
106#include <optional>
107#include <string>
108#include <utility>
109
110#define DEBUG_TYPE "instcombine"
112#include <optional>
113
114using namespace llvm;
115using namespace llvm::PatternMatch;
116
117STATISTIC(NumWorklistIterations,
118 "Number of instruction combining iterations performed");
119STATISTIC(NumOneIteration, "Number of functions with one iteration");
120STATISTIC(NumTwoIterations, "Number of functions with two iterations");
121STATISTIC(NumThreeIterations, "Number of functions with three iterations");
122STATISTIC(NumFourOrMoreIterations,
123 "Number of functions with four or more iterations");
124
125STATISTIC(NumCombined , "Number of insts combined");
126STATISTIC(NumConstProp, "Number of constant folds");
127STATISTIC(NumDeadInst , "Number of dead inst eliminated");
128STATISTIC(NumSunkInst , "Number of instructions sunk");
129STATISTIC(NumExpand, "Number of expansions");
130STATISTIC(NumFactor , "Number of factorizations");
131STATISTIC(NumReassoc , "Number of reassociations");
132DEBUG_COUNTER(VisitCounter, "instcombine-visit",
133 "Controls which instructions are visited");
134
135static cl::opt<bool> EnableCodeSinking("instcombine-code-sinking",
136 cl::desc("Enable code sinking"),
137 cl::init(true));
138
140 "instcombine-max-sink-users", cl::init(32),
141 cl::desc("Maximum number of undroppable users for instruction sinking"));
142
144MaxArraySize("instcombine-maxarray-size", cl::init(1024),
145 cl::desc("Maximum array size considered when doing a combine"));
146
147namespace llvm {
149} // end namespace llvm
150
151// FIXME: Remove this flag when it is no longer necessary to convert
152// llvm.dbg.declare to avoid inaccurate debug info. Setting this to false
153// increases variable availability at the cost of accuracy. Variables that
154// cannot be promoted by mem2reg or SROA will be described as living in memory
155// for their entire lifetime. However, passes like DSE and instcombine can
156// delete stores to the alloca, leading to misleading and inaccurate debug
157// information. This flag can be removed when those passes are fixed.
158static cl::opt<unsigned> ShouldLowerDbgDeclare("instcombine-lower-dbg-declare",
159 cl::Hidden, cl::init(true));
160
161std::optional<Instruction *>
163 // Handle target specific intrinsics
164 if (II.getCalledFunction()->isTargetIntrinsic()) {
165 return TTIForTargetIntrinsicsOnly.instCombineIntrinsic(*this, II);
166 }
167 return std::nullopt;
168}
169
171 IntrinsicInst &II, APInt DemandedMask, KnownBits &Known,
172 bool &KnownBitsComputed) {
173 // Handle target specific intrinsics
174 if (II.getCalledFunction()->isTargetIntrinsic()) {
175 return TTIForTargetIntrinsicsOnly.simplifyDemandedUseBitsIntrinsic(
176 *this, II, DemandedMask, Known, KnownBitsComputed);
177 }
178 return std::nullopt;
179}
180
182 IntrinsicInst &II, APInt DemandedElts, APInt &PoisonElts,
183 APInt &PoisonElts2, APInt &PoisonElts3,
184 std::function<void(Instruction *, unsigned, APInt, APInt &)>
185 SimplifyAndSetOp) {
186 // Handle target specific intrinsics
187 if (II.getCalledFunction()->isTargetIntrinsic()) {
188 return TTIForTargetIntrinsicsOnly.simplifyDemandedVectorEltsIntrinsic(
189 *this, II, DemandedElts, PoisonElts, PoisonElts2, PoisonElts3,
190 SimplifyAndSetOp);
191 }
192 return std::nullopt;
193}
194
195bool InstCombiner::isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const {
196 // Approved exception for TTI use: This queries a legality property of the
197 // target, not an profitability heuristic. Ideally this should be part of
198 // DataLayout instead.
199 return TTIForTargetIntrinsicsOnly.isValidAddrSpaceCast(FromAS, ToAS);
200}
201
202Value *InstCombinerImpl::EmitGEPOffset(GEPOperator *GEP, bool RewriteGEP) {
203 if (!RewriteGEP)
204 return llvm::emitGEPOffset(&Builder, DL, GEP);
205
206 IRBuilderBase::InsertPointGuard Guard(Builder);
207 auto *Inst = dyn_cast<Instruction>(GEP);
208 if (Inst)
209 Builder.SetInsertPoint(Inst);
210
211 Value *Offset = EmitGEPOffset(GEP);
212 // Rewrite non-trivial GEPs to avoid duplicating the offset arithmetic.
213 if (Inst && !GEP->hasAllConstantIndices() &&
214 !GEP->getSourceElementType()->isIntegerTy(8)) {
216 *Inst, Builder.CreateGEP(Builder.getInt8Ty(), GEP->getPointerOperand(),
217 Offset, "", GEP->getNoWrapFlags()));
219 }
220 return Offset;
221}
222
223Value *InstCombinerImpl::EmitGEPOffsets(ArrayRef<GEPOperator *> GEPs,
224 GEPNoWrapFlags NW, Type *IdxTy,
225 bool RewriteGEPs) {
226 auto Add = [&](Value *Sum, Value *Offset) -> Value * {
227 if (Sum)
228 return Builder.CreateAdd(Sum, Offset, "", NW.hasNoUnsignedWrap(),
229 NW.isInBounds());
230 else
231 return Offset;
232 };
233
234 Value *Sum = nullptr;
235 Value *OneUseSum = nullptr;
236 Value *OneUseBase = nullptr;
237 GEPNoWrapFlags OneUseFlags = GEPNoWrapFlags::all();
238 for (GEPOperator *GEP : reverse(GEPs)) {
239 Value *Offset;
240 {
241 // Expand the offset at the point of the previous GEP to enable rewriting.
242 // However, use the original insertion point for calculating Sum.
243 IRBuilderBase::InsertPointGuard Guard(Builder);
244 auto *Inst = dyn_cast<Instruction>(GEP);
245 if (RewriteGEPs && Inst)
246 Builder.SetInsertPoint(Inst);
247
249 if (Offset->getType() != IdxTy)
250 Offset = Builder.CreateVectorSplat(
251 cast<VectorType>(IdxTy)->getElementCount(), Offset);
252 if (GEP->hasOneUse()) {
253 // Offsets of one-use GEPs will be merged into the next multi-use GEP.
254 OneUseSum = Add(OneUseSum, Offset);
255 OneUseFlags = OneUseFlags.intersectForOffsetAdd(GEP->getNoWrapFlags());
256 if (!OneUseBase)
257 OneUseBase = GEP->getPointerOperand();
258 continue;
259 }
260
261 if (OneUseSum)
262 Offset = Add(OneUseSum, Offset);
263
264 // Rewrite the GEP to reuse the computed offset. This also includes
265 // offsets from preceding one-use GEPs.
266 if (RewriteGEPs && Inst &&
267 !(GEP->getSourceElementType()->isIntegerTy(8) &&
268 GEP->getOperand(1) == Offset)) {
270 *Inst,
271 Builder.CreatePtrAdd(
272 OneUseBase ? OneUseBase : GEP->getPointerOperand(), Offset, "",
273 OneUseFlags.intersectForOffsetAdd(GEP->getNoWrapFlags())));
275 }
276 }
277
278 Sum = Add(Sum, Offset);
279 OneUseSum = OneUseBase = nullptr;
280 OneUseFlags = GEPNoWrapFlags::all();
281 }
282 if (OneUseSum)
283 Sum = Add(Sum, OneUseSum);
284 if (!Sum)
285 return Constant::getNullValue(IdxTy);
286 return Sum;
287}
288
289/// Legal integers and common types are considered desirable. This is used to
290/// avoid creating instructions with types that may not be supported well by the
291/// the backend.
292/// NOTE: This treats i8, i16 and i32 specially because they are common
293/// types in frontend languages.
294bool InstCombinerImpl::isDesirableIntType(unsigned BitWidth) const {
295 switch (BitWidth) {
296 case 8:
297 case 16:
298 case 32:
299 return true;
300 default:
301 return DL.isLegalInteger(BitWidth);
302 }
303}
304
305/// Return true if it is desirable to convert an integer computation from a
306/// given bit width to a new bit width.
307/// We don't want to convert from a legal or desirable type (like i8) to an
308/// illegal type or from a smaller to a larger illegal type. A width of '1'
309/// is always treated as a desirable type because i1 is a fundamental type in
310/// IR, and there are many specialized optimizations for i1 types.
311/// Common/desirable widths are equally treated as legal to convert to, in
312/// order to open up more combining opportunities.
313bool InstCombinerImpl::shouldChangeType(unsigned FromWidth,
314 unsigned ToWidth) const {
315 bool FromLegal = FromWidth == 1 || DL.isLegalInteger(FromWidth);
316 bool ToLegal = ToWidth == 1 || DL.isLegalInteger(ToWidth);
317
318 // Convert to desirable widths even if they are not legal types.
319 // Only shrink types, to prevent infinite loops.
320 if (ToWidth < FromWidth && isDesirableIntType(ToWidth))
321 return true;
322
323 // If this is a legal or desiable integer from type, and the result would be
324 // an illegal type, don't do the transformation.
325 if ((FromLegal || isDesirableIntType(FromWidth)) && !ToLegal)
326 return false;
327
328 // Otherwise, if both are illegal, do not increase the size of the result. We
329 // do allow things like i160 -> i64, but not i64 -> i160.
330 if (!FromLegal && !ToLegal && ToWidth > FromWidth)
331 return false;
332
333 return true;
334}
335
336/// Return true if it is desirable to convert a computation from 'From' to 'To'.
337/// We don't want to convert from a legal to an illegal type or from a smaller
338/// to a larger illegal type. i1 is always treated as a legal type because it is
339/// a fundamental type in IR, and there are many specialized optimizations for
340/// i1 types.
341bool InstCombinerImpl::shouldChangeType(Type *From, Type *To) const {
342 // TODO: This could be extended to allow vectors. Datalayout changes might be
343 // needed to properly support that.
344 if (!From->isIntegerTy() || !To->isIntegerTy())
345 return false;
346
347 unsigned FromWidth = From->getPrimitiveSizeInBits();
348 unsigned ToWidth = To->getPrimitiveSizeInBits();
349 return shouldChangeType(FromWidth, ToWidth);
350}
351
352// Return true, if No Signed Wrap should be maintained for I.
353// The No Signed Wrap flag can be kept if the operation "B (I.getOpcode) C",
354// where both B and C should be ConstantInts, results in a constant that does
355// not overflow. This function only handles the Add/Sub/Mul opcodes. For
356// all other opcodes, the function conservatively returns false.
359 if (!OBO || !OBO->hasNoSignedWrap())
360 return false;
361
362 const APInt *BVal, *CVal;
363 if (!match(B, m_APInt(BVal)) || !match(C, m_APInt(CVal)))
364 return false;
365
366 // We reason about Add/Sub/Mul Only.
367 bool Overflow = false;
368 switch (I.getOpcode()) {
369 case Instruction::Add:
370 (void)BVal->sadd_ov(*CVal, Overflow);
371 break;
372 case Instruction::Sub:
373 (void)BVal->ssub_ov(*CVal, Overflow);
374 break;
375 case Instruction::Mul:
376 (void)BVal->smul_ov(*CVal, Overflow);
377 break;
378 default:
379 // Conservatively return false for other opcodes.
380 return false;
381 }
382 return !Overflow;
383}
384
387 return OBO && OBO->hasNoUnsignedWrap();
388}
389
392 return OBO && OBO->hasNoSignedWrap();
393}
394
395/// Conservatively clears subclassOptionalData after a reassociation or
396/// commutation. We preserve fast-math flags when applicable as they can be
397/// preserved.
400 if (!FPMO) {
401 I.clearSubclassOptionalData();
402 return;
403 }
404
405 FastMathFlags FMF = I.getFastMathFlags();
406 I.clearSubclassOptionalData();
407 I.setFastMathFlags(FMF);
408}
409
410/// Combine constant operands of associative operations either before or after a
411/// cast to eliminate one of the associative operations:
412/// (op (cast (op X, C2)), C1) --> (cast (op X, op (C1, C2)))
413/// (op (cast (op X, C2)), C1) --> (op (cast X), op (C1, C2))
415 InstCombinerImpl &IC) {
416 auto *Cast = dyn_cast<CastInst>(BinOp1->getOperand(0));
417 if (!Cast || !Cast->hasOneUse())
418 return false;
419
420 // TODO: Enhance logic for other casts and remove this check.
421 auto CastOpcode = Cast->getOpcode();
422 if (CastOpcode != Instruction::ZExt)
423 return false;
424
425 // TODO: Enhance logic for other BinOps and remove this check.
426 if (!BinOp1->isBitwiseLogicOp())
427 return false;
428
429 auto AssocOpcode = BinOp1->getOpcode();
430 auto *BinOp2 = dyn_cast<BinaryOperator>(Cast->getOperand(0));
431 if (!BinOp2 || !BinOp2->hasOneUse() || BinOp2->getOpcode() != AssocOpcode)
432 return false;
433
434 Constant *C1, *C2;
435 if (!match(BinOp1->getOperand(1), m_Constant(C1)) ||
436 !match(BinOp2->getOperand(1), m_Constant(C2)))
437 return false;
438
439 // TODO: This assumes a zext cast.
440 // Eg, if it was a trunc, we'd cast C1 to the source type because casting C2
441 // to the destination type might lose bits.
442
443 // Fold the constants together in the destination type:
444 // (op (cast (op X, C2)), C1) --> (op (cast X), FoldedC)
445 const DataLayout &DL = IC.getDataLayout();
446 Type *DestTy = C1->getType();
447 Constant *CastC2 = ConstantFoldCastOperand(CastOpcode, C2, DestTy, DL);
448 if (!CastC2)
449 return false;
450 Constant *FoldedC = ConstantFoldBinaryOpOperands(AssocOpcode, C1, CastC2, DL);
451 if (!FoldedC)
452 return false;
453
454 IC.replaceOperand(*Cast, 0, BinOp2->getOperand(0));
455 IC.replaceOperand(*BinOp1, 1, FoldedC);
457 Cast->dropPoisonGeneratingFlags();
458 return true;
459}
460
461// Simplifies IntToPtr/PtrToInt RoundTrip Cast.
462// inttoptr ( ptrtoint (x) ) --> x
463Value *InstCombinerImpl::simplifyIntToPtrRoundTripCast(Value *Val) {
464 auto *IntToPtr = dyn_cast<IntToPtrInst>(Val);
465 if (IntToPtr && DL.getTypeSizeInBits(IntToPtr->getDestTy()) ==
466 DL.getTypeSizeInBits(IntToPtr->getSrcTy())) {
467 auto *PtrToInt = dyn_cast<PtrToIntInst>(IntToPtr->getOperand(0));
468 Type *CastTy = IntToPtr->getDestTy();
469 if (PtrToInt &&
470 CastTy->getPointerAddressSpace() ==
471 PtrToInt->getSrcTy()->getPointerAddressSpace() &&
472 DL.getTypeSizeInBits(PtrToInt->getSrcTy()) ==
473 DL.getTypeSizeInBits(PtrToInt->getDestTy()))
474 return PtrToInt->getOperand(0);
475 }
476 return nullptr;
477}
478
479/// This performs a few simplifications for operators that are associative or
480/// commutative:
481///
482/// Commutative operators:
483///
484/// 1. Order operands such that they are listed from right (least complex) to
485/// left (most complex). This puts constants before unary operators before
486/// binary operators.
487///
488/// Associative operators:
489///
490/// 2. Transform: "(A op B) op C" ==> "A op (B op C)" if "B op C" simplifies.
491/// 3. Transform: "A op (B op C)" ==> "(A op B) op C" if "A op B" simplifies.
492///
493/// Associative and commutative operators:
494///
495/// 4. Transform: "(A op B) op C" ==> "(C op A) op B" if "C op A" simplifies.
496/// 5. Transform: "A op (B op C)" ==> "B op (C op A)" if "C op A" simplifies.
497/// 6. Transform: "(A op C1) op (B op C2)" ==> "(A op B) op (C1 op C2)"
498/// if C1 and C2 are constants.
500 Instruction::BinaryOps Opcode = I.getOpcode();
501 bool Changed = false;
502
503 do {
504 // Order operands such that they are listed from right (least complex) to
505 // left (most complex). This puts constants before unary operators before
506 // binary operators.
507 if (I.isCommutative() && getComplexity(I.getOperand(0)) <
508 getComplexity(I.getOperand(1)))
509 Changed = !I.swapOperands();
510
511 if (I.isCommutative()) {
512 if (auto Pair = matchSymmetricPair(I.getOperand(0), I.getOperand(1))) {
513 replaceOperand(I, 0, Pair->first);
514 replaceOperand(I, 1, Pair->second);
515 Changed = true;
516 }
517 }
518
519 BinaryOperator *Op0 = dyn_cast<BinaryOperator>(I.getOperand(0));
520 BinaryOperator *Op1 = dyn_cast<BinaryOperator>(I.getOperand(1));
521
522 if (I.isAssociative()) {
523 // Transform: "(A op B) op C" ==> "A op (B op C)" if "B op C" simplifies.
524 if (Op0 && Op0->getOpcode() == Opcode) {
525 Value *A = Op0->getOperand(0);
526 Value *B = Op0->getOperand(1);
527 Value *C = I.getOperand(1);
528
529 // Does "B op C" simplify?
530 if (Value *V = simplifyBinOp(Opcode, B, C, SQ.getWithInstruction(&I))) {
531 // It simplifies to V. Form "A op V".
532 replaceOperand(I, 0, A);
533 replaceOperand(I, 1, V);
534 bool IsNUW = hasNoUnsignedWrap(I) && hasNoUnsignedWrap(*Op0);
535 bool IsNSW = maintainNoSignedWrap(I, B, C) && hasNoSignedWrap(*Op0);
536
537 // Conservatively clear all optional flags since they may not be
538 // preserved by the reassociation. Reset nsw/nuw based on the above
539 // analysis.
541
542 // Note: this is only valid because SimplifyBinOp doesn't look at
543 // the operands to Op0.
544 if (IsNUW)
545 I.setHasNoUnsignedWrap(true);
546
547 if (IsNSW)
548 I.setHasNoSignedWrap(true);
549
550 Changed = true;
551 ++NumReassoc;
552 continue;
553 }
554 }
555
556 // Transform: "A op (B op C)" ==> "(A op B) op C" if "A op B" simplifies.
557 if (Op1 && Op1->getOpcode() == Opcode) {
558 Value *A = I.getOperand(0);
559 Value *B = Op1->getOperand(0);
560 Value *C = Op1->getOperand(1);
561
562 // Does "A op B" simplify?
563 if (Value *V = simplifyBinOp(Opcode, A, B, SQ.getWithInstruction(&I))) {
564 // It simplifies to V. Form "V op C".
565 replaceOperand(I, 0, V);
566 replaceOperand(I, 1, C);
567 // Conservatively clear the optional flags, since they may not be
568 // preserved by the reassociation.
570 Changed = true;
571 ++NumReassoc;
572 continue;
573 }
574 }
575 }
576
577 if (I.isAssociative() && I.isCommutative()) {
578 if (simplifyAssocCastAssoc(&I, *this)) {
579 Changed = true;
580 ++NumReassoc;
581 continue;
582 }
583
584 // Transform: "(A op B) op C" ==> "(C op A) op B" if "C op A" simplifies.
585 if (Op0 && Op0->getOpcode() == Opcode) {
586 Value *A = Op0->getOperand(0);
587 Value *B = Op0->getOperand(1);
588 Value *C = I.getOperand(1);
589
590 // Does "C op A" simplify?
591 if (Value *V = simplifyBinOp(Opcode, C, A, SQ.getWithInstruction(&I))) {
592 // It simplifies to V. Form "V op B".
593 replaceOperand(I, 0, V);
594 replaceOperand(I, 1, B);
595 // Conservatively clear the optional flags, since they may not be
596 // preserved by the reassociation.
598 Changed = true;
599 ++NumReassoc;
600 continue;
601 }
602 }
603
604 // Transform: "A op (B op C)" ==> "B op (C op A)" if "C op A" simplifies.
605 if (Op1 && Op1->getOpcode() == Opcode) {
606 Value *A = I.getOperand(0);
607 Value *B = Op1->getOperand(0);
608 Value *C = Op1->getOperand(1);
609
610 // Does "C op A" simplify?
611 if (Value *V = simplifyBinOp(Opcode, C, A, SQ.getWithInstruction(&I))) {
612 // It simplifies to V. Form "B op V".
613 replaceOperand(I, 0, B);
614 replaceOperand(I, 1, V);
615 // Conservatively clear the optional flags, since they may not be
616 // preserved by the reassociation.
618 Changed = true;
619 ++NumReassoc;
620 continue;
621 }
622 }
623
624 // Transform: "(A op C1) op (B op C2)" ==> "(A op B) op (C1 op C2)"
625 // if C1 and C2 are constants.
626 Value *A, *B;
627 Constant *C1, *C2, *CRes;
628 if (Op0 && Op1 &&
629 Op0->getOpcode() == Opcode && Op1->getOpcode() == Opcode &&
630 match(Op0, m_OneUse(m_BinOp(m_Value(A), m_Constant(C1)))) &&
631 match(Op1, m_OneUse(m_BinOp(m_Value(B), m_Constant(C2)))) &&
632 (CRes = ConstantFoldBinaryOpOperands(Opcode, C1, C2, DL))) {
633 bool IsNUW = hasNoUnsignedWrap(I) &&
634 hasNoUnsignedWrap(*Op0) &&
635 hasNoUnsignedWrap(*Op1);
636 BinaryOperator *NewBO = (IsNUW && Opcode == Instruction::Add) ?
637 BinaryOperator::CreateNUW(Opcode, A, B) :
638 BinaryOperator::Create(Opcode, A, B);
639
640 if (isa<FPMathOperator>(NewBO)) {
641 FastMathFlags Flags = I.getFastMathFlags() &
642 Op0->getFastMathFlags() &
643 Op1->getFastMathFlags();
644 NewBO->setFastMathFlags(Flags);
645 }
646 InsertNewInstWith(NewBO, I.getIterator());
647 NewBO->takeName(Op1);
648 replaceOperand(I, 0, NewBO);
649 replaceOperand(I, 1, CRes);
650 // Conservatively clear the optional flags, since they may not be
651 // preserved by the reassociation.
653 if (IsNUW)
654 I.setHasNoUnsignedWrap(true);
655
656 Changed = true;
657 continue;
658 }
659 }
660
661 // No further simplifications.
662 return Changed;
663 } while (true);
664}
665
666/// Return whether "X LOp (Y ROp Z)" is always equal to
667/// "(X LOp Y) ROp (X LOp Z)".
670 // X & (Y | Z) <--> (X & Y) | (X & Z)
671 // X & (Y ^ Z) <--> (X & Y) ^ (X & Z)
672 if (LOp == Instruction::And)
673 return ROp == Instruction::Or || ROp == Instruction::Xor;
674
675 // X | (Y & Z) <--> (X | Y) & (X | Z)
676 if (LOp == Instruction::Or)
677 return ROp == Instruction::And;
678
679 // X * (Y + Z) <--> (X * Y) + (X * Z)
680 // X * (Y - Z) <--> (X * Y) - (X * Z)
681 if (LOp == Instruction::Mul)
682 return ROp == Instruction::Add || ROp == Instruction::Sub;
683
684 return false;
685}
686
687/// Return whether "(X LOp Y) ROp Z" is always equal to
688/// "(X ROp Z) LOp (Y ROp Z)".
692 return leftDistributesOverRight(ROp, LOp);
693
694 // (X {&|^} Y) >> Z <--> (X >> Z) {&|^} (Y >> Z) for all shifts.
696
697 // TODO: It would be nice to handle division, aka "(X + Y)/Z = X/Z + Y/Z",
698 // but this requires knowing that the addition does not overflow and other
699 // such subtleties.
700}
701
702/// This function returns identity value for given opcode, which can be used to
703/// factor patterns like (X * 2) + X ==> (X * 2) + (X * 1) ==> X * (2 + 1).
705 if (isa<Constant>(V))
706 return nullptr;
707
708 return ConstantExpr::getBinOpIdentity(Opcode, V->getType());
709}
710
711/// This function predicates factorization using distributive laws. By default,
712/// it just returns the 'Op' inputs. But for special-cases like
713/// 'add(shl(X, 5), ...)', this function will have TopOpcode == Instruction::Add
714/// and Op = shl(X, 5). The 'shl' is treated as the more general 'mul X, 32' to
715/// allow more factorization opportunities.
718 Value *&LHS, Value *&RHS, BinaryOperator *OtherOp) {
719 assert(Op && "Expected a binary operator");
720 LHS = Op->getOperand(0);
721 RHS = Op->getOperand(1);
722 if (TopOpcode == Instruction::Add || TopOpcode == Instruction::Sub) {
723 Constant *C;
724 if (match(Op, m_Shl(m_Value(), m_ImmConstant(C)))) {
725 // X << C --> X * (1 << C)
727 Instruction::Shl, ConstantInt::get(Op->getType(), 1), C);
728 assert(RHS && "Constant folding of immediate constants failed");
729 return Instruction::Mul;
730 }
731 // TODO: We can add other conversions e.g. shr => div etc.
732 }
733 if (Instruction::isBitwiseLogicOp(TopOpcode)) {
734 if (OtherOp && OtherOp->getOpcode() == Instruction::AShr &&
736 // lshr nneg C, X --> ashr nneg C, X
737 return Instruction::AShr;
738 }
739 }
740 return Op->getOpcode();
741}
742
743/// This tries to simplify binary operations by factorizing out common terms
744/// (e. g. "(A*B)+(A*C)" -> "A*(B+C)").
747 Instruction::BinaryOps InnerOpcode, Value *A,
748 Value *B, Value *C, Value *D) {
749 assert(A && B && C && D && "All values must be provided");
750
751 Value *V = nullptr;
752 Value *RetVal = nullptr;
753 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
754 Instruction::BinaryOps TopLevelOpcode = I.getOpcode();
755
756 // Does "X op' Y" always equal "Y op' X"?
757 bool InnerCommutative = Instruction::isCommutative(InnerOpcode);
758
759 // Does "X op' (Y op Z)" always equal "(X op' Y) op (X op' Z)"?
760 if (leftDistributesOverRight(InnerOpcode, TopLevelOpcode)) {
761 // Does the instruction have the form "(A op' B) op (A op' D)" or, in the
762 // commutative case, "(A op' B) op (C op' A)"?
763 if (A == C || (InnerCommutative && A == D)) {
764 if (A != C)
765 std::swap(C, D);
766 // Consider forming "A op' (B op D)".
767 // If "B op D" simplifies then it can be formed with no cost.
768 V = simplifyBinOp(TopLevelOpcode, B, D, SQ.getWithInstruction(&I));
769
770 // If "B op D" doesn't simplify then only go on if one of the existing
771 // operations "A op' B" and "C op' D" will be zapped as no longer used.
772 if (!V && (LHS->hasOneUse() || RHS->hasOneUse()))
773 V = Builder.CreateBinOp(TopLevelOpcode, B, D, RHS->getName());
774 if (V)
775 RetVal = Builder.CreateBinOp(InnerOpcode, A, V);
776 }
777 }
778
779 // Does "(X op Y) op' Z" always equal "(X op' Z) op (Y op' Z)"?
780 if (!RetVal && rightDistributesOverLeft(TopLevelOpcode, InnerOpcode)) {
781 // Does the instruction have the form "(A op' B) op (C op' B)" or, in the
782 // commutative case, "(A op' B) op (B op' D)"?
783 if (B == D || (InnerCommutative && B == C)) {
784 if (B != D)
785 std::swap(C, D);
786 // Consider forming "(A op C) op' B".
787 // If "A op C" simplifies then it can be formed with no cost.
788 V = simplifyBinOp(TopLevelOpcode, A, C, SQ.getWithInstruction(&I));
789
790 // If "A op C" doesn't simplify then only go on if one of the existing
791 // operations "A op' B" and "C op' D" will be zapped as no longer used.
792 if (!V && (LHS->hasOneUse() || RHS->hasOneUse()))
793 V = Builder.CreateBinOp(TopLevelOpcode, A, C, LHS->getName());
794 if (V)
795 RetVal = Builder.CreateBinOp(InnerOpcode, V, B);
796 }
797 }
798
799 if (!RetVal)
800 return nullptr;
801
802 ++NumFactor;
803 RetVal->takeName(&I);
804
805 // Try to add no-overflow flags to the final value.
806 if (isa<BinaryOperator>(RetVal)) {
807 bool HasNSW = false;
808 bool HasNUW = false;
810 HasNSW = I.hasNoSignedWrap();
811 HasNUW = I.hasNoUnsignedWrap();
812 }
813 if (auto *LOBO = dyn_cast<OverflowingBinaryOperator>(LHS)) {
814 HasNSW &= LOBO->hasNoSignedWrap();
815 HasNUW &= LOBO->hasNoUnsignedWrap();
816 }
817
818 if (auto *ROBO = dyn_cast<OverflowingBinaryOperator>(RHS)) {
819 HasNSW &= ROBO->hasNoSignedWrap();
820 HasNUW &= ROBO->hasNoUnsignedWrap();
821 }
822
823 if (TopLevelOpcode == Instruction::Add && InnerOpcode == Instruction::Mul) {
824 // We can propagate 'nsw' if we know that
825 // %Y = mul nsw i16 %X, C
826 // %Z = add nsw i16 %Y, %X
827 // =>
828 // %Z = mul nsw i16 %X, C+1
829 //
830 // iff C+1 isn't INT_MIN
831 const APInt *CInt;
832 if (match(V, m_APInt(CInt)) && !CInt->isMinSignedValue())
833 cast<Instruction>(RetVal)->setHasNoSignedWrap(HasNSW);
834
835 // nuw can be propagated with any constant or nuw value.
836 cast<Instruction>(RetVal)->setHasNoUnsignedWrap(HasNUW);
837 }
838 }
839 return RetVal;
840}
841
842// If `I` has one Const operand and the other matches `(ctpop (not x))`,
843// replace `(ctpop (not x))` with `(sub nuw nsw BitWidth(x), (ctpop x))`.
844// This is only useful is the new subtract can fold so we only handle the
845// following cases:
846// 1) (add/sub/disjoint_or C, (ctpop (not x))
847// -> (add/sub/disjoint_or C', (ctpop x))
848// 1) (cmp pred C, (ctpop (not x))
849// -> (cmp pred C', (ctpop x))
851 unsigned Opc = I->getOpcode();
852 unsigned ConstIdx = 1;
853 switch (Opc) {
854 default:
855 return nullptr;
856 // (ctpop (not x)) <-> (sub nuw nsw BitWidth(x) - (ctpop x))
857 // We can fold the BitWidth(x) with add/sub/icmp as long the other operand
858 // is constant.
859 case Instruction::Sub:
860 ConstIdx = 0;
861 break;
862 case Instruction::ICmp:
863 // Signed predicates aren't correct in some edge cases like for i2 types, as
864 // well since (ctpop x) is known [0, log2(BitWidth(x))] almost all signed
865 // comparisons against it are simplfied to unsigned.
866 if (cast<ICmpInst>(I)->isSigned())
867 return nullptr;
868 break;
869 case Instruction::Or:
870 if (!match(I, m_DisjointOr(m_Value(), m_Value())))
871 return nullptr;
872 [[fallthrough]];
873 case Instruction::Add:
874 break;
875 }
876
877 Value *Op;
878 // Find ctpop.
879 if (!match(I->getOperand(1 - ConstIdx),
881 return nullptr;
882
883 Constant *C;
884 // Check other operand is ImmConstant.
885 if (!match(I->getOperand(ConstIdx), m_ImmConstant(C)))
886 return nullptr;
887
888 Type *Ty = Op->getType();
889 Constant *BitWidthC = ConstantInt::get(Ty, Ty->getScalarSizeInBits());
890 // Need extra check for icmp. Note if this check is true, it generally means
891 // the icmp will simplify to true/false.
892 if (Opc == Instruction::ICmp && !cast<ICmpInst>(I)->isEquality()) {
893 Constant *Cmp =
895 if (!Cmp || !Cmp->isZeroValue())
896 return nullptr;
897 }
898
899 // Check we can invert `(not x)` for free.
900 bool Consumes = false;
901 if (!isFreeToInvert(Op, Op->hasOneUse(), Consumes) || !Consumes)
902 return nullptr;
903 Value *NotOp = getFreelyInverted(Op, Op->hasOneUse(), &Builder);
904 assert(NotOp != nullptr &&
905 "Desync between isFreeToInvert and getFreelyInverted");
906
907 Value *CtpopOfNotOp = Builder.CreateIntrinsic(Ty, Intrinsic::ctpop, NotOp);
908
909 Value *R = nullptr;
910
911 // Do the transformation here to avoid potentially introducing an infinite
912 // loop.
913 switch (Opc) {
914 case Instruction::Sub:
915 R = Builder.CreateAdd(CtpopOfNotOp, ConstantExpr::getSub(C, BitWidthC));
916 break;
917 case Instruction::Or:
918 case Instruction::Add:
919 R = Builder.CreateSub(ConstantExpr::getAdd(C, BitWidthC), CtpopOfNotOp);
920 break;
921 case Instruction::ICmp:
922 R = Builder.CreateICmp(cast<ICmpInst>(I)->getSwappedPredicate(),
923 CtpopOfNotOp, ConstantExpr::getSub(BitWidthC, C));
924 break;
925 default:
926 llvm_unreachable("Unhandled Opcode");
927 }
928 assert(R != nullptr);
929 return replaceInstUsesWith(*I, R);
930}
931
932// (Binop1 (Binop2 (logic_shift X, C), C1), (logic_shift Y, C))
933// IFF
934// 1) the logic_shifts match
935// 2) either both binops are binops and one is `and` or
936// BinOp1 is `and`
937// (logic_shift (inv_logic_shift C1, C), C) == C1 or
938//
939// -> (logic_shift (Binop1 (Binop2 X, inv_logic_shift(C1, C)), Y), C)
940//
941// (Binop1 (Binop2 (logic_shift X, Amt), Mask), (logic_shift Y, Amt))
942// IFF
943// 1) the logic_shifts match
944// 2) BinOp1 == BinOp2 (if BinOp == `add`, then also requires `shl`).
945//
946// -> (BinOp (logic_shift (BinOp X, Y)), Mask)
947//
948// (Binop1 (Binop2 (arithmetic_shift X, Amt), Mask), (arithmetic_shift Y, Amt))
949// IFF
950// 1) Binop1 is bitwise logical operator `and`, `or` or `xor`
951// 2) Binop2 is `not`
952//
953// -> (arithmetic_shift Binop1((not X), Y), Amt)
954
956 const DataLayout &DL = I.getDataLayout();
957 auto IsValidBinOpc = [](unsigned Opc) {
958 switch (Opc) {
959 default:
960 return false;
961 case Instruction::And:
962 case Instruction::Or:
963 case Instruction::Xor:
964 case Instruction::Add:
965 // Skip Sub as we only match constant masks which will canonicalize to use
966 // add.
967 return true;
968 }
969 };
970
971 // Check if we can distribute binop arbitrarily. `add` + `lshr` has extra
972 // constraints.
973 auto IsCompletelyDistributable = [](unsigned BinOpc1, unsigned BinOpc2,
974 unsigned ShOpc) {
975 assert(ShOpc != Instruction::AShr);
976 return (BinOpc1 != Instruction::Add && BinOpc2 != Instruction::Add) ||
977 ShOpc == Instruction::Shl;
978 };
979
980 auto GetInvShift = [](unsigned ShOpc) {
981 assert(ShOpc != Instruction::AShr);
982 return ShOpc == Instruction::LShr ? Instruction::Shl : Instruction::LShr;
983 };
984
985 auto CanDistributeBinops = [&](unsigned BinOpc1, unsigned BinOpc2,
986 unsigned ShOpc, Constant *CMask,
987 Constant *CShift) {
988 // If the BinOp1 is `and` we don't need to check the mask.
989 if (BinOpc1 == Instruction::And)
990 return true;
991
992 // For all other possible transfers we need complete distributable
993 // binop/shift (anything but `add` + `lshr`).
994 if (!IsCompletelyDistributable(BinOpc1, BinOpc2, ShOpc))
995 return false;
996
997 // If BinOp2 is `and`, any mask works (this only really helps for non-splat
998 // vecs, otherwise the mask will be simplified and the following check will
999 // handle it).
1000 if (BinOpc2 == Instruction::And)
1001 return true;
1002
1003 // Otherwise, need mask that meets the below requirement.
1004 // (logic_shift (inv_logic_shift Mask, ShAmt), ShAmt) == Mask
1005 Constant *MaskInvShift =
1006 ConstantFoldBinaryOpOperands(GetInvShift(ShOpc), CMask, CShift, DL);
1007 return ConstantFoldBinaryOpOperands(ShOpc, MaskInvShift, CShift, DL) ==
1008 CMask;
1009 };
1010
1011 auto MatchBinOp = [&](unsigned ShOpnum) -> Instruction * {
1012 Constant *CMask, *CShift;
1013 Value *X, *Y, *ShiftedX, *Mask, *Shift;
1014 if (!match(I.getOperand(ShOpnum),
1015 m_OneUse(m_Shift(m_Value(Y), m_Value(Shift)))))
1016 return nullptr;
1017 if (!match(I.getOperand(1 - ShOpnum),
1019 m_OneUse(m_Shift(m_Value(X), m_Specific(Shift))),
1020 m_Value(ShiftedX)),
1021 m_Value(Mask))))
1022 return nullptr;
1023 // Make sure we are matching instruction shifts and not ConstantExpr
1024 auto *IY = dyn_cast<Instruction>(I.getOperand(ShOpnum));
1025 auto *IX = dyn_cast<Instruction>(ShiftedX);
1026 if (!IY || !IX)
1027 return nullptr;
1028
1029 // LHS and RHS need same shift opcode
1030 unsigned ShOpc = IY->getOpcode();
1031 if (ShOpc != IX->getOpcode())
1032 return nullptr;
1033
1034 // Make sure binop is real instruction and not ConstantExpr
1035 auto *BO2 = dyn_cast<Instruction>(I.getOperand(1 - ShOpnum));
1036 if (!BO2)
1037 return nullptr;
1038
1039 unsigned BinOpc = BO2->getOpcode();
1040 // Make sure we have valid binops.
1041 if (!IsValidBinOpc(I.getOpcode()) || !IsValidBinOpc(BinOpc))
1042 return nullptr;
1043
1044 if (ShOpc == Instruction::AShr) {
1045 if (Instruction::isBitwiseLogicOp(I.getOpcode()) &&
1046 BinOpc == Instruction::Xor && match(Mask, m_AllOnes())) {
1047 Value *NotX = Builder.CreateNot(X);
1048 Value *NewBinOp = Builder.CreateBinOp(I.getOpcode(), Y, NotX);
1050 static_cast<Instruction::BinaryOps>(ShOpc), NewBinOp, Shift);
1051 }
1052
1053 return nullptr;
1054 }
1055
1056 // If BinOp1 == BinOp2 and it's bitwise or shl with add, then just
1057 // distribute to drop the shift irrelevant of constants.
1058 if (BinOpc == I.getOpcode() &&
1059 IsCompletelyDistributable(I.getOpcode(), BinOpc, ShOpc)) {
1060 Value *NewBinOp2 = Builder.CreateBinOp(I.getOpcode(), X, Y);
1061 Value *NewBinOp1 = Builder.CreateBinOp(
1062 static_cast<Instruction::BinaryOps>(ShOpc), NewBinOp2, Shift);
1063 return BinaryOperator::Create(I.getOpcode(), NewBinOp1, Mask);
1064 }
1065
1066 // Otherwise we can only distribute by constant shifting the mask, so
1067 // ensure we have constants.
1068 if (!match(Shift, m_ImmConstant(CShift)))
1069 return nullptr;
1070 if (!match(Mask, m_ImmConstant(CMask)))
1071 return nullptr;
1072
1073 // Check if we can distribute the binops.
1074 if (!CanDistributeBinops(I.getOpcode(), BinOpc, ShOpc, CMask, CShift))
1075 return nullptr;
1076
1077 Constant *NewCMask =
1078 ConstantFoldBinaryOpOperands(GetInvShift(ShOpc), CMask, CShift, DL);
1079 Value *NewBinOp2 = Builder.CreateBinOp(
1080 static_cast<Instruction::BinaryOps>(BinOpc), X, NewCMask);
1081 Value *NewBinOp1 = Builder.CreateBinOp(I.getOpcode(), Y, NewBinOp2);
1082 return BinaryOperator::Create(static_cast<Instruction::BinaryOps>(ShOpc),
1083 NewBinOp1, CShift);
1084 };
1085
1086 if (Instruction *R = MatchBinOp(0))
1087 return R;
1088 return MatchBinOp(1);
1089}
1090
1091// (Binop (zext C), (select C, T, F))
1092// -> (select C, (binop 1, T), (binop 0, F))
1093//
1094// (Binop (sext C), (select C, T, F))
1095// -> (select C, (binop -1, T), (binop 0, F))
1096//
1097// Attempt to simplify binary operations into a select with folded args, when
1098// one operand of the binop is a select instruction and the other operand is a
1099// zext/sext extension, whose value is the select condition.
1102 // TODO: this simplification may be extended to any speculatable instruction,
1103 // not just binops, and would possibly be handled better in FoldOpIntoSelect.
1104 Instruction::BinaryOps Opc = I.getOpcode();
1105 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
1106 Value *A, *CondVal, *TrueVal, *FalseVal;
1107 Value *CastOp;
1108
1109 auto MatchSelectAndCast = [&](Value *CastOp, Value *SelectOp) {
1110 return match(CastOp, m_ZExtOrSExt(m_Value(A))) &&
1111 A->getType()->getScalarSizeInBits() == 1 &&
1112 match(SelectOp, m_Select(m_Value(CondVal), m_Value(TrueVal),
1113 m_Value(FalseVal)));
1114 };
1115
1116 // Make sure one side of the binop is a select instruction, and the other is a
1117 // zero/sign extension operating on a i1.
1118 if (MatchSelectAndCast(LHS, RHS))
1119 CastOp = LHS;
1120 else if (MatchSelectAndCast(RHS, LHS))
1121 CastOp = RHS;
1122 else
1123 return nullptr;
1124
1125 auto NewFoldedConst = [&](bool IsTrueArm, Value *V) {
1126 bool IsCastOpRHS = (CastOp == RHS);
1127 bool IsZExt = isa<ZExtInst>(CastOp);
1128 Constant *C;
1129
1130 if (IsTrueArm) {
1131 C = Constant::getNullValue(V->getType());
1132 } else if (IsZExt) {
1133 unsigned BitWidth = V->getType()->getScalarSizeInBits();
1134 C = Constant::getIntegerValue(V->getType(), APInt(BitWidth, 1));
1135 } else {
1136 C = Constant::getAllOnesValue(V->getType());
1137 }
1138
1139 return IsCastOpRHS ? Builder.CreateBinOp(Opc, V, C)
1140 : Builder.CreateBinOp(Opc, C, V);
1141 };
1142
1143 // If the value used in the zext/sext is the select condition, or the negated
1144 // of the select condition, the binop can be simplified.
1145 if (CondVal == A) {
1146 Value *NewTrueVal = NewFoldedConst(false, TrueVal);
1147 return SelectInst::Create(CondVal, NewTrueVal,
1148 NewFoldedConst(true, FalseVal));
1149 }
1150
1151 if (match(A, m_Not(m_Specific(CondVal)))) {
1152 Value *NewTrueVal = NewFoldedConst(true, TrueVal);
1153 return SelectInst::Create(CondVal, NewTrueVal,
1154 NewFoldedConst(false, FalseVal));
1155 }
1156
1157 return nullptr;
1158}
1159
1161 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
1164 Instruction::BinaryOps TopLevelOpcode = I.getOpcode();
1165 Value *A, *B, *C, *D;
1166 Instruction::BinaryOps LHSOpcode, RHSOpcode;
1167
1168 if (Op0)
1169 LHSOpcode = getBinOpsForFactorization(TopLevelOpcode, Op0, A, B, Op1);
1170 if (Op1)
1171 RHSOpcode = getBinOpsForFactorization(TopLevelOpcode, Op1, C, D, Op0);
1172
1173 // The instruction has the form "(A op' B) op (C op' D)". Try to factorize
1174 // a common term.
1175 if (Op0 && Op1 && LHSOpcode == RHSOpcode)
1176 if (Value *V = tryFactorization(I, SQ, Builder, LHSOpcode, A, B, C, D))
1177 return V;
1178
1179 // The instruction has the form "(A op' B) op (C)". Try to factorize common
1180 // term.
1181 if (Op0)
1182 if (Value *Ident = getIdentityValue(LHSOpcode, RHS))
1183 if (Value *V =
1184 tryFactorization(I, SQ, Builder, LHSOpcode, A, B, RHS, Ident))
1185 return V;
1186
1187 // The instruction has the form "(B) op (C op' D)". Try to factorize common
1188 // term.
1189 if (Op1)
1190 if (Value *Ident = getIdentityValue(RHSOpcode, LHS))
1191 if (Value *V =
1192 tryFactorization(I, SQ, Builder, RHSOpcode, LHS, Ident, C, D))
1193 return V;
1194
1195 return nullptr;
1196}
1197
1198/// This tries to simplify binary operations which some other binary operation
1199/// distributes over either by factorizing out common terms
1200/// (eg "(A*B)+(A*C)" -> "A*(B+C)") or expanding out if this results in
1201/// simplifications (eg: "A & (B | C) -> (A&B) | (A&C)" if this is a win).
1202/// Returns the simplified value, or null if it didn't simplify.
1204 Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
1207 Instruction::BinaryOps TopLevelOpcode = I.getOpcode();
1208
1209 // Factorization.
1210 if (Value *R = tryFactorizationFolds(I))
1211 return R;
1212
1213 // Expansion.
1214 if (Op0 && rightDistributesOverLeft(Op0->getOpcode(), TopLevelOpcode)) {
1215 // The instruction has the form "(A op' B) op C". See if expanding it out
1216 // to "(A op C) op' (B op C)" results in simplifications.
1217 Value *A = Op0->getOperand(0), *B = Op0->getOperand(1), *C = RHS;
1218 Instruction::BinaryOps InnerOpcode = Op0->getOpcode(); // op'
1219
1220 // Disable the use of undef because it's not safe to distribute undef.
1221 auto SQDistributive = SQ.getWithInstruction(&I).getWithoutUndef();
1222 Value *L = simplifyBinOp(TopLevelOpcode, A, C, SQDistributive);
1223 Value *R = simplifyBinOp(TopLevelOpcode, B, C, SQDistributive);
1224
1225 // Do "A op C" and "B op C" both simplify?
1226 if (L && R) {
1227 // They do! Return "L op' R".
1228 ++NumExpand;
1229 C = Builder.CreateBinOp(InnerOpcode, L, R);
1230 C->takeName(&I);
1231 return C;
1232 }
1233
1234 // Does "A op C" simplify to the identity value for the inner opcode?
1235 if (L && L == ConstantExpr::getBinOpIdentity(InnerOpcode, L->getType())) {
1236 // They do! Return "B op C".
1237 ++NumExpand;
1238 C = Builder.CreateBinOp(TopLevelOpcode, B, C);
1239 C->takeName(&I);
1240 return C;
1241 }
1242
1243 // Does "B op C" simplify to the identity value for the inner opcode?
1244 if (R && R == ConstantExpr::getBinOpIdentity(InnerOpcode, R->getType())) {
1245 // They do! Return "A op C".
1246 ++NumExpand;
1247 C = Builder.CreateBinOp(TopLevelOpcode, A, C);
1248 C->takeName(&I);
1249 return C;
1250 }
1251 }
1252
1253 if (Op1 && leftDistributesOverRight(TopLevelOpcode, Op1->getOpcode())) {
1254 // The instruction has the form "A op (B op' C)". See if expanding it out
1255 // to "(A op B) op' (A op C)" results in simplifications.
1256 Value *A = LHS, *B = Op1->getOperand(0), *C = Op1->getOperand(1);
1257 Instruction::BinaryOps InnerOpcode = Op1->getOpcode(); // op'
1258
1259 // Disable the use of undef because it's not safe to distribute undef.
1260 auto SQDistributive = SQ.getWithInstruction(&I).getWithoutUndef();
1261 Value *L = simplifyBinOp(TopLevelOpcode, A, B, SQDistributive);
1262 Value *R = simplifyBinOp(TopLevelOpcode, A, C, SQDistributive);
1263
1264 // Do "A op B" and "A op C" both simplify?
1265 if (L && R) {
1266 // They do! Return "L op' R".
1267 ++NumExpand;
1268 A = Builder.CreateBinOp(InnerOpcode, L, R);
1269 A->takeName(&I);
1270 return A;
1271 }
1272
1273 // Does "A op B" simplify to the identity value for the inner opcode?
1274 if (L && L == ConstantExpr::getBinOpIdentity(InnerOpcode, L->getType())) {
1275 // They do! Return "A op C".
1276 ++NumExpand;
1277 A = Builder.CreateBinOp(TopLevelOpcode, A, C);
1278 A->takeName(&I);
1279 return A;
1280 }
1281
1282 // Does "A op C" simplify to the identity value for the inner opcode?
1283 if (R && R == ConstantExpr::getBinOpIdentity(InnerOpcode, R->getType())) {
1284 // They do! Return "A op B".
1285 ++NumExpand;
1286 A = Builder.CreateBinOp(TopLevelOpcode, A, B);
1287 A->takeName(&I);
1288 return A;
1289 }
1290 }
1291
1292 return SimplifySelectsFeedingBinaryOp(I, LHS, RHS);
1293}
1294
1295static std::optional<std::pair<Value *, Value *>>
1297 if (LHS->getParent() != RHS->getParent())
1298 return std::nullopt;
1299
1300 if (LHS->getNumIncomingValues() < 2)
1301 return std::nullopt;
1302
1303 if (!equal(LHS->blocks(), RHS->blocks()))
1304 return std::nullopt;
1305
1306 Value *L0 = LHS->getIncomingValue(0);
1307 Value *R0 = RHS->getIncomingValue(0);
1308
1309 for (unsigned I = 1, E = LHS->getNumIncomingValues(); I != E; ++I) {
1310 Value *L1 = LHS->getIncomingValue(I);
1311 Value *R1 = RHS->getIncomingValue(I);
1312
1313 if ((L0 == L1 && R0 == R1) || (L0 == R1 && R0 == L1))
1314 continue;
1315
1316 return std::nullopt;
1317 }
1318
1319 return std::optional(std::pair(L0, R0));
1320}
1321
1322std::optional<std::pair<Value *, Value *>>
1323InstCombinerImpl::matchSymmetricPair(Value *LHS, Value *RHS) {
1326 if (!LHSInst || !RHSInst || LHSInst->getOpcode() != RHSInst->getOpcode())
1327 return std::nullopt;
1328 switch (LHSInst->getOpcode()) {
1329 case Instruction::PHI:
1331 case Instruction::Select: {
1332 Value *Cond = LHSInst->getOperand(0);
1333 Value *TrueVal = LHSInst->getOperand(1);
1334 Value *FalseVal = LHSInst->getOperand(2);
1335 if (Cond == RHSInst->getOperand(0) && TrueVal == RHSInst->getOperand(2) &&
1336 FalseVal == RHSInst->getOperand(1))
1337 return std::pair(TrueVal, FalseVal);
1338 return std::nullopt;
1339 }
1340 case Instruction::Call: {
1341 // Match min(a, b) and max(a, b)
1342 MinMaxIntrinsic *LHSMinMax = dyn_cast<MinMaxIntrinsic>(LHSInst);
1343 MinMaxIntrinsic *RHSMinMax = dyn_cast<MinMaxIntrinsic>(RHSInst);
1344 if (LHSMinMax && RHSMinMax &&
1345 LHSMinMax->getPredicate() ==
1347 ((LHSMinMax->getLHS() == RHSMinMax->getLHS() &&
1348 LHSMinMax->getRHS() == RHSMinMax->getRHS()) ||
1349 (LHSMinMax->getLHS() == RHSMinMax->getRHS() &&
1350 LHSMinMax->getRHS() == RHSMinMax->getLHS())))
1351 return std::pair(LHSMinMax->getLHS(), LHSMinMax->getRHS());
1352 return std::nullopt;
1353 }
1354 default:
1355 return std::nullopt;
1356 }
1357}
1358
1360 Value *LHS,
1361 Value *RHS) {
1362 Value *A, *B, *C, *D, *E, *F;
1363 bool LHSIsSelect = match(LHS, m_Select(m_Value(A), m_Value(B), m_Value(C)));
1364 bool RHSIsSelect = match(RHS, m_Select(m_Value(D), m_Value(E), m_Value(F)));
1365 if (!LHSIsSelect && !RHSIsSelect)
1366 return nullptr;
1367
1369 ? nullptr
1370 : cast<SelectInst>(LHSIsSelect ? LHS : RHS);
1371
1372 FastMathFlags FMF;
1374 if (isa<FPMathOperator>(&I)) {
1375 FMF = I.getFastMathFlags();
1376 Builder.setFastMathFlags(FMF);
1377 }
1378
1379 Instruction::BinaryOps Opcode = I.getOpcode();
1380 SimplifyQuery Q = SQ.getWithInstruction(&I);
1381
1382 Value *Cond, *True = nullptr, *False = nullptr;
1383
1384 // Special-case for add/negate combination. Replace the zero in the negation
1385 // with the trailing add operand:
1386 // (Cond ? TVal : -N) + Z --> Cond ? True : (Z - N)
1387 // (Cond ? -N : FVal) + Z --> Cond ? (Z - N) : False
1388 auto foldAddNegate = [&](Value *TVal, Value *FVal, Value *Z) -> Value * {
1389 // We need an 'add' and exactly 1 arm of the select to have been simplified.
1390 if (Opcode != Instruction::Add || (!True && !False) || (True && False))
1391 return nullptr;
1392 Value *N;
1393 if (True && match(FVal, m_Neg(m_Value(N)))) {
1394 Value *Sub = Builder.CreateSub(Z, N);
1395 return Builder.CreateSelect(Cond, True, Sub, I.getName(), SI);
1396 }
1397 if (False && match(TVal, m_Neg(m_Value(N)))) {
1398 Value *Sub = Builder.CreateSub(Z, N);
1399 return Builder.CreateSelect(Cond, Sub, False, I.getName(), SI);
1400 }
1401 return nullptr;
1402 };
1403
1404 if (LHSIsSelect && RHSIsSelect && A == D) {
1405 // (A ? B : C) op (A ? E : F) -> A ? (B op E) : (C op F)
1406 Cond = A;
1407 True = simplifyBinOp(Opcode, B, E, FMF, Q);
1408 False = simplifyBinOp(Opcode, C, F, FMF, Q);
1409
1410 if (LHS->hasOneUse() && RHS->hasOneUse()) {
1411 if (False && !True)
1412 True = Builder.CreateBinOp(Opcode, B, E);
1413 else if (True && !False)
1414 False = Builder.CreateBinOp(Opcode, C, F);
1415 }
1416 } else if (LHSIsSelect && LHS->hasOneUse()) {
1417 // (A ? B : C) op Y -> A ? (B op Y) : (C op Y)
1418 Cond = A;
1419 True = simplifyBinOp(Opcode, B, RHS, FMF, Q);
1420 False = simplifyBinOp(Opcode, C, RHS, FMF, Q);
1421 if (Value *NewSel = foldAddNegate(B, C, RHS))
1422 return NewSel;
1423 } else if (RHSIsSelect && RHS->hasOneUse()) {
1424 // X op (D ? E : F) -> D ? (X op E) : (X op F)
1425 Cond = D;
1426 True = simplifyBinOp(Opcode, LHS, E, FMF, Q);
1427 False = simplifyBinOp(Opcode, LHS, F, FMF, Q);
1428 if (Value *NewSel = foldAddNegate(E, F, LHS))
1429 return NewSel;
1430 }
1431
1432 if (!True || !False)
1433 return nullptr;
1434
1435 Value *NewSI = Builder.CreateSelect(Cond, True, False, I.getName(), SI);
1436 NewSI->takeName(&I);
1437 return NewSI;
1438}
1439
1440/// Freely adapt every user of V as-if V was changed to !V.
1441/// WARNING: only if canFreelyInvertAllUsersOf() said this can be done.
1443 assert(!isa<Constant>(I) && "Shouldn't invert users of constant");
1444 for (User *U : make_early_inc_range(I->users())) {
1445 if (U == IgnoredUser)
1446 continue; // Don't consider this user.
1447 switch (cast<Instruction>(U)->getOpcode()) {
1448 case Instruction::Select: {
1449 auto *SI = cast<SelectInst>(U);
1450 SI->swapValues();
1451 SI->swapProfMetadata();
1452 break;
1453 }
1454 case Instruction::Br: {
1456 BI->swapSuccessors(); // swaps prof metadata too
1457 if (BPI)
1458 BPI->swapSuccEdgesProbabilities(BI->getParent());
1459 break;
1460 }
1461 case Instruction::Xor:
1463 // Add to worklist for DCE.
1465 break;
1466 default:
1467 llvm_unreachable("Got unexpected user - out of sync with "
1468 "canFreelyInvertAllUsersOf() ?");
1469 }
1470 }
1471
1472 // Update pre-existing debug value uses.
1473 SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
1474 llvm::findDbgValues(I, DbgVariableRecords);
1475
1476 for (DbgVariableRecord *DbgVal : DbgVariableRecords) {
1477 SmallVector<uint64_t, 1> Ops = {dwarf::DW_OP_not};
1478 for (unsigned Idx = 0, End = DbgVal->getNumVariableLocationOps();
1479 Idx != End; ++Idx)
1480 if (DbgVal->getVariableLocationOp(Idx) == I)
1481 DbgVal->setExpression(
1482 DIExpression::appendOpsToArg(DbgVal->getExpression(), Ops, Idx));
1483 }
1484}
1485
1486/// Given a 'sub' instruction, return the RHS of the instruction if the LHS is a
1487/// constant zero (which is the 'negate' form).
1488Value *InstCombinerImpl::dyn_castNegVal(Value *V) const {
1489 Value *NegV;
1490 if (match(V, m_Neg(m_Value(NegV))))
1491 return NegV;
1492
1493 // Constants can be considered to be negated values if they can be folded.
1495 return ConstantExpr::getNeg(C);
1496
1498 if (C->getType()->getElementType()->isIntegerTy())
1499 return ConstantExpr::getNeg(C);
1500
1502 for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
1503 Constant *Elt = CV->getAggregateElement(i);
1504 if (!Elt)
1505 return nullptr;
1506
1507 if (isa<UndefValue>(Elt))
1508 continue;
1509
1510 if (!isa<ConstantInt>(Elt))
1511 return nullptr;
1512 }
1513 return ConstantExpr::getNeg(CV);
1514 }
1515
1516 // Negate integer vector splats.
1517 if (auto *CV = dyn_cast<Constant>(V))
1518 if (CV->getType()->isVectorTy() &&
1519 CV->getType()->getScalarType()->isIntegerTy() && CV->getSplatValue())
1520 return ConstantExpr::getNeg(CV);
1521
1522 return nullptr;
1523}
1524
1525// Try to fold:
1526// 1) (fp_binop ({s|u}itofp x), ({s|u}itofp y))
1527// -> ({s|u}itofp (int_binop x, y))
1528// 2) (fp_binop ({s|u}itofp x), FpC)
1529// -> ({s|u}itofp (int_binop x, (fpto{s|u}i FpC)))
1530//
1531// Assuming the sign of the cast for x/y is `OpsFromSigned`.
1532Instruction *InstCombinerImpl::foldFBinOpOfIntCastsFromSign(
1533 BinaryOperator &BO, bool OpsFromSigned, std::array<Value *, 2> IntOps,
1535
1536 Type *FPTy = BO.getType();
1537 Type *IntTy = IntOps[0]->getType();
1538
1539 unsigned IntSz = IntTy->getScalarSizeInBits();
1540 // This is the maximum number of inuse bits by the integer where the int -> fp
1541 // casts are exact.
1542 unsigned MaxRepresentableBits =
1544
1545 // Preserve known number of leading bits. This can allow us to trivial nsw/nuw
1546 // checks later on.
1547 unsigned NumUsedLeadingBits[2] = {IntSz, IntSz};
1548
1549 // NB: This only comes up if OpsFromSigned is true, so there is no need to
1550 // cache if between calls to `foldFBinOpOfIntCastsFromSign`.
1551 auto IsNonZero = [&](unsigned OpNo) -> bool {
1552 if (OpsKnown[OpNo].hasKnownBits() &&
1553 OpsKnown[OpNo].getKnownBits(SQ).isNonZero())
1554 return true;
1555 return isKnownNonZero(IntOps[OpNo], SQ);
1556 };
1557
1558 auto IsNonNeg = [&](unsigned OpNo) -> bool {
1559 // NB: This matches the impl in ValueTracking, we just try to use cached
1560 // knownbits here. If we ever start supporting WithCache for
1561 // `isKnownNonNegative`, change this to an explicit call.
1562 return OpsKnown[OpNo].getKnownBits(SQ).isNonNegative();
1563 };
1564
1565 // Check if we know for certain that ({s|u}itofp op) is exact.
1566 auto IsValidPromotion = [&](unsigned OpNo) -> bool {
1567 // Can we treat this operand as the desired sign?
1568 if (OpsFromSigned != isa<SIToFPInst>(BO.getOperand(OpNo)) &&
1569 !IsNonNeg(OpNo))
1570 return false;
1571
1572 // If fp precision >= bitwidth(op) then its exact.
1573 // NB: This is slightly conservative for `sitofp`. For signed conversion, we
1574 // can handle `MaxRepresentableBits == IntSz - 1` as the sign bit will be
1575 // handled specially. We can't, however, increase the bound arbitrarily for
1576 // `sitofp` as for larger sizes, it won't sign extend.
1577 if (MaxRepresentableBits < IntSz) {
1578 // Otherwise if its signed cast check that fp precisions >= bitwidth(op) -
1579 // numSignBits(op).
1580 // TODO: If we add support for `WithCache` in `ComputeNumSignBits`, change
1581 // `IntOps[OpNo]` arguments to `KnownOps[OpNo]`.
1582 if (OpsFromSigned)
1583 NumUsedLeadingBits[OpNo] = IntSz - ComputeNumSignBits(IntOps[OpNo]);
1584 // Finally for unsigned check that fp precision >= bitwidth(op) -
1585 // numLeadingZeros(op).
1586 else {
1587 NumUsedLeadingBits[OpNo] =
1588 IntSz - OpsKnown[OpNo].getKnownBits(SQ).countMinLeadingZeros();
1589 }
1590 }
1591 // NB: We could also check if op is known to be a power of 2 or zero (which
1592 // will always be representable). Its unlikely, however, that is we are
1593 // unable to bound op in any way we will be able to pass the overflow checks
1594 // later on.
1595
1596 if (MaxRepresentableBits < NumUsedLeadingBits[OpNo])
1597 return false;
1598 // Signed + Mul also requires that op is non-zero to avoid -0 cases.
1599 return !OpsFromSigned || BO.getOpcode() != Instruction::FMul ||
1600 IsNonZero(OpNo);
1601 };
1602
1603 // If we have a constant rhs, see if we can losslessly convert it to an int.
1604 if (Op1FpC != nullptr) {
1605 // Signed + Mul req non-zero
1606 if (OpsFromSigned && BO.getOpcode() == Instruction::FMul &&
1607 !match(Op1FpC, m_NonZeroFP()))
1608 return nullptr;
1609
1611 OpsFromSigned ? Instruction::FPToSI : Instruction::FPToUI, Op1FpC,
1612 IntTy, DL);
1613 if (Op1IntC == nullptr)
1614 return nullptr;
1615 if (ConstantFoldCastOperand(OpsFromSigned ? Instruction::SIToFP
1616 : Instruction::UIToFP,
1617 Op1IntC, FPTy, DL) != Op1FpC)
1618 return nullptr;
1619
1620 // First try to keep sign of cast the same.
1621 IntOps[1] = Op1IntC;
1622 }
1623
1624 // Ensure lhs/rhs integer types match.
1625 if (IntTy != IntOps[1]->getType())
1626 return nullptr;
1627
1628 if (Op1FpC == nullptr) {
1629 if (!IsValidPromotion(1))
1630 return nullptr;
1631 }
1632 if (!IsValidPromotion(0))
1633 return nullptr;
1634
1635 // Final we check if the integer version of the binop will not overflow.
1637 // Because of the precision check, we can often rule out overflows.
1638 bool NeedsOverflowCheck = true;
1639 // Try to conservatively rule out overflow based on the already done precision
1640 // checks.
1641 unsigned OverflowMaxOutputBits = OpsFromSigned ? 2 : 1;
1642 unsigned OverflowMaxCurBits =
1643 std::max(NumUsedLeadingBits[0], NumUsedLeadingBits[1]);
1644 bool OutputSigned = OpsFromSigned;
1645 switch (BO.getOpcode()) {
1646 case Instruction::FAdd:
1647 IntOpc = Instruction::Add;
1648 OverflowMaxOutputBits += OverflowMaxCurBits;
1649 break;
1650 case Instruction::FSub:
1651 IntOpc = Instruction::Sub;
1652 OverflowMaxOutputBits += OverflowMaxCurBits;
1653 break;
1654 case Instruction::FMul:
1655 IntOpc = Instruction::Mul;
1656 OverflowMaxOutputBits += OverflowMaxCurBits * 2;
1657 break;
1658 default:
1659 llvm_unreachable("Unsupported binop");
1660 }
1661 // The precision check may have already ruled out overflow.
1662 if (OverflowMaxOutputBits < IntSz) {
1663 NeedsOverflowCheck = false;
1664 // We can bound unsigned overflow from sub to in range signed value (this is
1665 // what allows us to avoid the overflow check for sub).
1666 if (IntOpc == Instruction::Sub)
1667 OutputSigned = true;
1668 }
1669
1670 // Precision check did not rule out overflow, so need to check.
1671 // TODO: If we add support for `WithCache` in `willNotOverflow`, change
1672 // `IntOps[...]` arguments to `KnownOps[...]`.
1673 if (NeedsOverflowCheck &&
1674 !willNotOverflow(IntOpc, IntOps[0], IntOps[1], BO, OutputSigned))
1675 return nullptr;
1676
1677 Value *IntBinOp = Builder.CreateBinOp(IntOpc, IntOps[0], IntOps[1]);
1678 if (auto *IntBO = dyn_cast<BinaryOperator>(IntBinOp)) {
1679 IntBO->setHasNoSignedWrap(OutputSigned);
1680 IntBO->setHasNoUnsignedWrap(!OutputSigned);
1681 }
1682 if (OutputSigned)
1683 return new SIToFPInst(IntBinOp, FPTy);
1684 return new UIToFPInst(IntBinOp, FPTy);
1685}
1686
1687// Try to fold:
1688// 1) (fp_binop ({s|u}itofp x), ({s|u}itofp y))
1689// -> ({s|u}itofp (int_binop x, y))
1690// 2) (fp_binop ({s|u}itofp x), FpC)
1691// -> ({s|u}itofp (int_binop x, (fpto{s|u}i FpC)))
1692Instruction *InstCombinerImpl::foldFBinOpOfIntCasts(BinaryOperator &BO) {
1693 // Don't perform the fold on vectors, as the integer operation may be much
1694 // more expensive than the float operation in that case.
1695 if (BO.getType()->isVectorTy())
1696 return nullptr;
1697
1698 std::array<Value *, 2> IntOps = {nullptr, nullptr};
1699 Constant *Op1FpC = nullptr;
1700 // Check for:
1701 // 1) (binop ({s|u}itofp x), ({s|u}itofp y))
1702 // 2) (binop ({s|u}itofp x), FpC)
1703 if (!match(BO.getOperand(0), m_SIToFP(m_Value(IntOps[0]))) &&
1704 !match(BO.getOperand(0), m_UIToFP(m_Value(IntOps[0]))))
1705 return nullptr;
1706
1707 if (!match(BO.getOperand(1), m_Constant(Op1FpC)) &&
1708 !match(BO.getOperand(1), m_SIToFP(m_Value(IntOps[1]))) &&
1709 !match(BO.getOperand(1), m_UIToFP(m_Value(IntOps[1]))))
1710 return nullptr;
1711
1712 // Cache KnownBits a bit to potentially save some analysis.
1713 SmallVector<WithCache<const Value *>, 2> OpsKnown = {IntOps[0], IntOps[1]};
1714
1715 // Try treating x/y as coming from both `uitofp` and `sitofp`. There are
1716 // different constraints depending on the sign of the cast.
1717 // NB: `(uitofp nneg X)` == `(sitofp nneg X)`.
1718 if (Instruction *R = foldFBinOpOfIntCastsFromSign(BO, /*OpsFromSigned=*/false,
1719 IntOps, Op1FpC, OpsKnown))
1720 return R;
1721 return foldFBinOpOfIntCastsFromSign(BO, /*OpsFromSigned=*/true, IntOps,
1722 Op1FpC, OpsKnown);
1723}
1724
1725/// A binop with a constant operand and a sign-extended boolean operand may be
1726/// converted into a select of constants by applying the binary operation to
1727/// the constant with the two possible values of the extended boolean (0 or -1).
1728Instruction *InstCombinerImpl::foldBinopOfSextBoolToSelect(BinaryOperator &BO) {
1729 // TODO: Handle non-commutative binop (constant is operand 0).
1730 // TODO: Handle zext.
1731 // TODO: Peek through 'not' of cast.
1732 Value *BO0 = BO.getOperand(0);
1733 Value *BO1 = BO.getOperand(1);
1734 Value *X;
1735 Constant *C;
1736 if (!match(BO0, m_SExt(m_Value(X))) || !match(BO1, m_ImmConstant(C)) ||
1737 !X->getType()->isIntOrIntVectorTy(1))
1738 return nullptr;
1739
1740 // bo (sext i1 X), C --> select X, (bo -1, C), (bo 0, C)
1743 Value *TVal = Builder.CreateBinOp(BO.getOpcode(), Ones, C);
1744 Value *FVal = Builder.CreateBinOp(BO.getOpcode(), Zero, C);
1745 return createSelectInstWithUnknownProfile(X, TVal, FVal);
1746}
1747
1749 bool IsTrueArm) {
1751 for (Value *Op : I.operands()) {
1752 Value *V = nullptr;
1753 if (Op == SI) {
1754 V = IsTrueArm ? SI->getTrueValue() : SI->getFalseValue();
1755 } else if (match(SI->getCondition(),
1758 m_Specific(Op), m_Value(V))) &&
1760 // Pass
1761 } else {
1762 V = Op;
1763 }
1764 Ops.push_back(V);
1765 }
1766
1767 return simplifyInstructionWithOperands(&I, Ops, I.getDataLayout());
1768}
1769
1771 Value *NewOp, InstCombiner &IC) {
1772 Instruction *Clone = I.clone();
1773 Clone->replaceUsesOfWith(SI, NewOp);
1775 IC.InsertNewInstBefore(Clone, I.getIterator());
1776 return Clone;
1777}
1778
1780 bool FoldWithMultiUse) {
1781 // Don't modify shared select instructions unless set FoldWithMultiUse
1782 if (!SI->hasOneUse() && !FoldWithMultiUse)
1783 return nullptr;
1784
1785 Value *TV = SI->getTrueValue();
1786 Value *FV = SI->getFalseValue();
1787
1788 // Bool selects with constant operands can be folded to logical ops.
1789 if (SI->getType()->isIntOrIntVectorTy(1))
1790 return nullptr;
1791
1792 // Avoid breaking min/max reduction pattern,
1793 // which is necessary for vectorization later.
1795 for (Value *IntrinOp : Op.operands())
1796 if (auto *PN = dyn_cast<PHINode>(IntrinOp))
1797 for (Value *PhiOp : PN->operands())
1798 if (PhiOp == &Op)
1799 return nullptr;
1800
1801 // Test if a FCmpInst instruction is used exclusively by a select as
1802 // part of a minimum or maximum operation. If so, refrain from doing
1803 // any other folding. This helps out other analyses which understand
1804 // non-obfuscated minimum and maximum idioms. And in this case, at
1805 // least one of the comparison operands has at least one user besides
1806 // the compare (the select), which would often largely negate the
1807 // benefit of folding anyway.
1808 if (auto *CI = dyn_cast<FCmpInst>(SI->getCondition())) {
1809 if (CI->hasOneUse()) {
1810 Value *Op0 = CI->getOperand(0), *Op1 = CI->getOperand(1);
1811 if (((TV == Op0 && FV == Op1) || (FV == Op0 && TV == Op1)) &&
1812 !CI->isCommutative())
1813 return nullptr;
1814 }
1815 }
1816
1817 // Make sure that one of the select arms folds successfully.
1818 Value *NewTV = simplifyOperationIntoSelectOperand(Op, SI, /*IsTrueArm=*/true);
1819 Value *NewFV =
1820 simplifyOperationIntoSelectOperand(Op, SI, /*IsTrueArm=*/false);
1821 if (!NewTV && !NewFV)
1822 return nullptr;
1823
1824 // Create an instruction for the arm that did not fold.
1825 if (!NewTV)
1826 NewTV = foldOperationIntoSelectOperand(Op, SI, TV, *this);
1827 if (!NewFV)
1828 NewFV = foldOperationIntoSelectOperand(Op, SI, FV, *this);
1829 return SelectInst::Create(SI->getCondition(), NewTV, NewFV, "", nullptr, SI);
1830}
1831
1833 Value *InValue, BasicBlock *InBB,
1834 const DataLayout &DL,
1835 const SimplifyQuery SQ) {
1836 // NB: It is a precondition of this transform that the operands be
1837 // phi translatable!
1839 for (Value *Op : I.operands()) {
1840 if (Op == PN)
1841 Ops.push_back(InValue);
1842 else
1843 Ops.push_back(Op->DoPHITranslation(PN->getParent(), InBB));
1844 }
1845
1846 // Don't consider the simplification successful if we get back a constant
1847 // expression. That's just an instruction in hiding.
1848 // Also reject the case where we simplify back to the phi node. We wouldn't
1849 // be able to remove it in that case.
1851 &I, Ops, SQ.getWithInstruction(InBB->getTerminator()));
1852 if (NewVal && NewVal != PN && !match(NewVal, m_ConstantExpr()))
1853 return NewVal;
1854
1855 // Check if incoming PHI value can be replaced with constant
1856 // based on implied condition.
1857 BranchInst *TerminatorBI = dyn_cast<BranchInst>(InBB->getTerminator());
1858 const ICmpInst *ICmp = dyn_cast<ICmpInst>(&I);
1859 if (TerminatorBI && TerminatorBI->isConditional() &&
1860 TerminatorBI->getSuccessor(0) != TerminatorBI->getSuccessor(1) && ICmp) {
1861 bool LHSIsTrue = TerminatorBI->getSuccessor(0) == PN->getParent();
1862 std::optional<bool> ImpliedCond = isImpliedCondition(
1863 TerminatorBI->getCondition(), ICmp->getCmpPredicate(), Ops[0], Ops[1],
1864 DL, LHSIsTrue);
1865 if (ImpliedCond)
1866 return ConstantInt::getBool(I.getType(), ImpliedCond.value());
1867 }
1868
1869 return nullptr;
1870}
1871
1873 bool AllowMultipleUses) {
1874 unsigned NumPHIValues = PN->getNumIncomingValues();
1875 if (NumPHIValues == 0)
1876 return nullptr;
1877
1878 // We normally only transform phis with a single use. However, if a PHI has
1879 // multiple uses and they are all the same operation, we can fold *all* of the
1880 // uses into the PHI.
1881 bool OneUse = PN->hasOneUse();
1882 bool IdenticalUsers = false;
1883 if (!AllowMultipleUses && !OneUse) {
1884 // Walk the use list for the instruction, comparing them to I.
1885 for (User *U : PN->users()) {
1887 if (UI != &I && !I.isIdenticalTo(UI))
1888 return nullptr;
1889 }
1890 // Otherwise, we can replace *all* users with the new PHI we form.
1891 IdenticalUsers = true;
1892 }
1893
1894 // Check that all operands are phi-translatable.
1895 for (Value *Op : I.operands()) {
1896 if (Op == PN)
1897 continue;
1898
1899 // Non-instructions never require phi-translation.
1900 auto *I = dyn_cast<Instruction>(Op);
1901 if (!I)
1902 continue;
1903
1904 // Phi-translate can handle phi nodes in the same block.
1905 if (isa<PHINode>(I))
1906 if (I->getParent() == PN->getParent())
1907 continue;
1908
1909 // Operand dominates the block, no phi-translation necessary.
1910 if (DT.dominates(I, PN->getParent()))
1911 continue;
1912
1913 // Not phi-translatable, bail out.
1914 return nullptr;
1915 }
1916
1917 // Check to see whether the instruction can be folded into each phi operand.
1918 // If there is one operand that does not fold, remember the BB it is in.
1919 SmallVector<Value *> NewPhiValues;
1920 SmallVector<unsigned int> OpsToMoveUseToIncomingBB;
1921 bool SeenNonSimplifiedInVal = false;
1922 for (unsigned i = 0; i != NumPHIValues; ++i) {
1923 Value *InVal = PN->getIncomingValue(i);
1924 BasicBlock *InBB = PN->getIncomingBlock(i);
1925
1926 if (auto *NewVal = simplifyInstructionWithPHI(I, PN, InVal, InBB, DL, SQ)) {
1927 NewPhiValues.push_back(NewVal);
1928 continue;
1929 }
1930
1931 // Handle some cases that can't be fully simplified, but where we know that
1932 // the two instructions will fold into one.
1933 auto WillFold = [&]() {
1934 if (!InVal->hasUseList() || !InVal->hasOneUser())
1935 return false;
1936
1937 // icmp of ucmp/scmp with constant will fold to icmp.
1938 const APInt *Ignored;
1939 if (isa<CmpIntrinsic>(InVal) &&
1940 match(&I, m_ICmp(m_Specific(PN), m_APInt(Ignored))))
1941 return true;
1942
1943 // icmp eq zext(bool), 0 will fold to !bool.
1944 if (isa<ZExtInst>(InVal) &&
1945 cast<ZExtInst>(InVal)->getSrcTy()->isIntOrIntVectorTy(1) &&
1946 match(&I,
1948 return true;
1949
1950 return false;
1951 };
1952
1953 if (WillFold()) {
1954 OpsToMoveUseToIncomingBB.push_back(i);
1955 NewPhiValues.push_back(nullptr);
1956 continue;
1957 }
1958
1959 if (!OneUse && !IdenticalUsers)
1960 return nullptr;
1961
1962 if (SeenNonSimplifiedInVal)
1963 return nullptr; // More than one non-simplified value.
1964 SeenNonSimplifiedInVal = true;
1965
1966 // If there is exactly one non-simplified value, we can insert a copy of the
1967 // operation in that block. However, if this is a critical edge, we would
1968 // be inserting the computation on some other paths (e.g. inside a loop).
1969 // Only do this if the pred block is unconditionally branching into the phi
1970 // block. Also, make sure that the pred block is not dead code.
1972 if (!BI || !BI->isUnconditional() || !DT.isReachableFromEntry(InBB))
1973 return nullptr;
1974
1975 NewPhiValues.push_back(nullptr);
1976 OpsToMoveUseToIncomingBB.push_back(i);
1977
1978 // Do not push the operation across a loop backedge. This could result in
1979 // an infinite combine loop, and is generally non-profitable (especially
1980 // if the operation was originally outside the loop).
1981 if (isBackEdge(InBB, PN->getParent()))
1982 return nullptr;
1983 }
1984
1985 // Clone the instruction that uses the phi node and move it into the incoming
1986 // BB because we know that the next iteration of InstCombine will simplify it.
1988 for (auto OpIndex : OpsToMoveUseToIncomingBB) {
1990 BasicBlock *OpBB = PN->getIncomingBlock(OpIndex);
1991
1992 Instruction *Clone = Clones.lookup(OpBB);
1993 if (!Clone) {
1994 Clone = I.clone();
1995 for (Use &U : Clone->operands()) {
1996 if (U == PN)
1997 U = Op;
1998 else
1999 U = U->DoPHITranslation(PN->getParent(), OpBB);
2000 }
2001 Clone = InsertNewInstBefore(Clone, OpBB->getTerminator()->getIterator());
2002 Clones.insert({OpBB, Clone});
2003 // We may have speculated the instruction.
2005 }
2006
2007 NewPhiValues[OpIndex] = Clone;
2008 }
2009
2010 // Okay, we can do the transformation: create the new PHI node.
2011 PHINode *NewPN = PHINode::Create(I.getType(), PN->getNumIncomingValues());
2012 InsertNewInstBefore(NewPN, PN->getIterator());
2013 NewPN->takeName(PN);
2014 NewPN->setDebugLoc(PN->getDebugLoc());
2015
2016 for (unsigned i = 0; i != NumPHIValues; ++i)
2017 NewPN->addIncoming(NewPhiValues[i], PN->getIncomingBlock(i));
2018
2019 if (IdenticalUsers) {
2020 // Collect and deduplicate users up-front to avoid iterator invalidation.
2022 for (User *U : PN->users()) {
2024 if (User == &I)
2025 continue;
2026 ToReplace.insert(User);
2027 }
2028 for (Instruction *I : ToReplace) {
2029 replaceInstUsesWith(*I, NewPN);
2031 }
2032 OneUse = true;
2033 }
2034
2035 if (OneUse) {
2036 replaceAllDbgUsesWith(*PN, *NewPN, *PN, DT);
2037 }
2038 return replaceInstUsesWith(I, NewPN);
2039}
2040
2042 if (!BO.isAssociative())
2043 return nullptr;
2044
2045 // Find the interleaved binary ops.
2046 auto Opc = BO.getOpcode();
2047 auto *BO0 = dyn_cast<BinaryOperator>(BO.getOperand(0));
2048 auto *BO1 = dyn_cast<BinaryOperator>(BO.getOperand(1));
2049 if (!BO0 || !BO1 || !BO0->hasNUses(2) || !BO1->hasNUses(2) ||
2050 BO0->getOpcode() != Opc || BO1->getOpcode() != Opc ||
2051 !BO0->isAssociative() || !BO1->isAssociative() ||
2052 BO0->getParent() != BO1->getParent())
2053 return nullptr;
2054
2055 assert(BO.isCommutative() && BO0->isCommutative() && BO1->isCommutative() &&
2056 "Expected commutative instructions!");
2057
2058 // Find the matching phis, forming the recurrences.
2059 PHINode *PN0, *PN1;
2060 Value *Start0, *Step0, *Start1, *Step1;
2061 if (!matchSimpleRecurrence(BO0, PN0, Start0, Step0) || !PN0->hasOneUse() ||
2062 !matchSimpleRecurrence(BO1, PN1, Start1, Step1) || !PN1->hasOneUse() ||
2063 PN0->getParent() != PN1->getParent())
2064 return nullptr;
2065
2066 assert(PN0->getNumIncomingValues() == 2 && PN1->getNumIncomingValues() == 2 &&
2067 "Expected PHIs with two incoming values!");
2068
2069 // Convert the start and step values to constants.
2070 auto *Init0 = dyn_cast<Constant>(Start0);
2071 auto *Init1 = dyn_cast<Constant>(Start1);
2072 auto *C0 = dyn_cast<Constant>(Step0);
2073 auto *C1 = dyn_cast<Constant>(Step1);
2074 if (!Init0 || !Init1 || !C0 || !C1)
2075 return nullptr;
2076
2077 // Fold the recurrence constants.
2078 auto *Init = ConstantFoldBinaryInstruction(Opc, Init0, Init1);
2079 auto *C = ConstantFoldBinaryInstruction(Opc, C0, C1);
2080 if (!Init || !C)
2081 return nullptr;
2082
2083 // Create the reduced PHI.
2084 auto *NewPN = PHINode::Create(PN0->getType(), PN0->getNumIncomingValues(),
2085 "reduced.phi");
2086
2087 // Create the new binary op.
2088 auto *NewBO = BinaryOperator::Create(Opc, NewPN, C);
2089 if (Opc == Instruction::FAdd || Opc == Instruction::FMul) {
2090 // Intersect FMF flags for FADD and FMUL.
2091 FastMathFlags Intersect = BO0->getFastMathFlags() &
2092 BO1->getFastMathFlags() & BO.getFastMathFlags();
2093 NewBO->setFastMathFlags(Intersect);
2094 } else {
2095 OverflowTracking Flags;
2096 Flags.AllKnownNonNegative = false;
2097 Flags.AllKnownNonZero = false;
2098 Flags.mergeFlags(*BO0);
2099 Flags.mergeFlags(*BO1);
2100 Flags.mergeFlags(BO);
2101 Flags.applyFlags(*NewBO);
2102 }
2103 NewBO->takeName(&BO);
2104
2105 for (unsigned I = 0, E = PN0->getNumIncomingValues(); I != E; ++I) {
2106 auto *V = PN0->getIncomingValue(I);
2107 auto *BB = PN0->getIncomingBlock(I);
2108 if (V == Init0) {
2109 assert(((PN1->getIncomingValue(0) == Init1 &&
2110 PN1->getIncomingBlock(0) == BB) ||
2111 (PN1->getIncomingValue(1) == Init1 &&
2112 PN1->getIncomingBlock(1) == BB)) &&
2113 "Invalid incoming block!");
2114 NewPN->addIncoming(Init, BB);
2115 } else if (V == BO0) {
2116 assert(((PN1->getIncomingValue(0) == BO1 &&
2117 PN1->getIncomingBlock(0) == BB) ||
2118 (PN1->getIncomingValue(1) == BO1 &&
2119 PN1->getIncomingBlock(1) == BB)) &&
2120 "Invalid incoming block!");
2121 NewPN->addIncoming(NewBO, BB);
2122 } else
2123 llvm_unreachable("Unexpected incoming value!");
2124 }
2125
2126 LLVM_DEBUG(dbgs() << " Combined " << *PN0 << "\n " << *BO0
2127 << "\n with " << *PN1 << "\n " << *BO1
2128 << '\n');
2129
2130 // Insert the new recurrence and remove the old (dead) ones.
2131 InsertNewInstWith(NewPN, PN0->getIterator());
2132 InsertNewInstWith(NewBO, BO0->getIterator());
2133
2140
2141 return replaceInstUsesWith(BO, NewBO);
2142}
2143
2145 // Attempt to fold binary operators whose operands are simple recurrences.
2146 if (auto *NewBO = foldBinopWithRecurrence(BO))
2147 return NewBO;
2148
2149 // TODO: This should be similar to the incoming values check in foldOpIntoPhi:
2150 // we are guarding against replicating the binop in >1 predecessor.
2151 // This could miss matching a phi with 2 constant incoming values.
2152 auto *Phi0 = dyn_cast<PHINode>(BO.getOperand(0));
2153 auto *Phi1 = dyn_cast<PHINode>(BO.getOperand(1));
2154 if (!Phi0 || !Phi1 || !Phi0->hasOneUse() || !Phi1->hasOneUse() ||
2155 Phi0->getNumOperands() != Phi1->getNumOperands())
2156 return nullptr;
2157
2158 // TODO: Remove the restriction for binop being in the same block as the phis.
2159 if (BO.getParent() != Phi0->getParent() ||
2160 BO.getParent() != Phi1->getParent())
2161 return nullptr;
2162
2163 // Fold if there is at least one specific constant value in phi0 or phi1's
2164 // incoming values that comes from the same block and this specific constant
2165 // value can be used to do optimization for specific binary operator.
2166 // For example:
2167 // %phi0 = phi i32 [0, %bb0], [%i, %bb1]
2168 // %phi1 = phi i32 [%j, %bb0], [0, %bb1]
2169 // %add = add i32 %phi0, %phi1
2170 // ==>
2171 // %add = phi i32 [%j, %bb0], [%i, %bb1]
2173 /*AllowRHSConstant*/ false);
2174 if (C) {
2175 SmallVector<Value *, 4> NewIncomingValues;
2176 auto CanFoldIncomingValuePair = [&](std::tuple<Use &, Use &> T) {
2177 auto &Phi0Use = std::get<0>(T);
2178 auto &Phi1Use = std::get<1>(T);
2179 if (Phi0->getIncomingBlock(Phi0Use) != Phi1->getIncomingBlock(Phi1Use))
2180 return false;
2181 Value *Phi0UseV = Phi0Use.get();
2182 Value *Phi1UseV = Phi1Use.get();
2183 if (Phi0UseV == C)
2184 NewIncomingValues.push_back(Phi1UseV);
2185 else if (Phi1UseV == C)
2186 NewIncomingValues.push_back(Phi0UseV);
2187 else
2188 return false;
2189 return true;
2190 };
2191
2192 if (all_of(zip(Phi0->operands(), Phi1->operands()),
2193 CanFoldIncomingValuePair)) {
2194 PHINode *NewPhi =
2195 PHINode::Create(Phi0->getType(), Phi0->getNumOperands());
2196 assert(NewIncomingValues.size() == Phi0->getNumOperands() &&
2197 "The number of collected incoming values should equal the number "
2198 "of the original PHINode operands!");
2199 for (unsigned I = 0; I < Phi0->getNumOperands(); I++)
2200 NewPhi->addIncoming(NewIncomingValues[I], Phi0->getIncomingBlock(I));
2201 return NewPhi;
2202 }
2203 }
2204
2205 if (Phi0->getNumOperands() != 2 || Phi1->getNumOperands() != 2)
2206 return nullptr;
2207
2208 // Match a pair of incoming constants for one of the predecessor blocks.
2209 BasicBlock *ConstBB, *OtherBB;
2210 Constant *C0, *C1;
2211 if (match(Phi0->getIncomingValue(0), m_ImmConstant(C0))) {
2212 ConstBB = Phi0->getIncomingBlock(0);
2213 OtherBB = Phi0->getIncomingBlock(1);
2214 } else if (match(Phi0->getIncomingValue(1), m_ImmConstant(C0))) {
2215 ConstBB = Phi0->getIncomingBlock(1);
2216 OtherBB = Phi0->getIncomingBlock(0);
2217 } else {
2218 return nullptr;
2219 }
2220 if (!match(Phi1->getIncomingValueForBlock(ConstBB), m_ImmConstant(C1)))
2221 return nullptr;
2222
2223 // The block that we are hoisting to must reach here unconditionally.
2224 // Otherwise, we could be speculatively executing an expensive or
2225 // non-speculative op.
2226 auto *PredBlockBranch = dyn_cast<BranchInst>(OtherBB->getTerminator());
2227 if (!PredBlockBranch || PredBlockBranch->isConditional() ||
2228 !DT.isReachableFromEntry(OtherBB))
2229 return nullptr;
2230
2231 // TODO: This check could be tightened to only apply to binops (div/rem) that
2232 // are not safe to speculatively execute. But that could allow hoisting
2233 // potentially expensive instructions (fdiv for example).
2234 for (auto BBIter = BO.getParent()->begin(); &*BBIter != &BO; ++BBIter)
2236 return nullptr;
2237
2238 // Fold constants for the predecessor block with constant incoming values.
2239 Constant *NewC = ConstantFoldBinaryOpOperands(BO.getOpcode(), C0, C1, DL);
2240 if (!NewC)
2241 return nullptr;
2242
2243 // Make a new binop in the predecessor block with the non-constant incoming
2244 // values.
2245 Builder.SetInsertPoint(PredBlockBranch);
2246 Value *NewBO = Builder.CreateBinOp(BO.getOpcode(),
2247 Phi0->getIncomingValueForBlock(OtherBB),
2248 Phi1->getIncomingValueForBlock(OtherBB));
2249 if (auto *NotFoldedNewBO = dyn_cast<BinaryOperator>(NewBO))
2250 NotFoldedNewBO->copyIRFlags(&BO);
2251
2252 // Replace the binop with a phi of the new values. The old phis are dead.
2253 PHINode *NewPhi = PHINode::Create(BO.getType(), 2);
2254 NewPhi->addIncoming(NewBO, OtherBB);
2255 NewPhi->addIncoming(NewC, ConstBB);
2256 return NewPhi;
2257}
2258
2260 if (!isa<Constant>(I.getOperand(1)))
2261 return nullptr;
2262
2263 if (auto *Sel = dyn_cast<SelectInst>(I.getOperand(0))) {
2264 if (Instruction *NewSel = FoldOpIntoSelect(I, Sel))
2265 return NewSel;
2266 } else if (auto *PN = dyn_cast<PHINode>(I.getOperand(0))) {
2267 if (Instruction *NewPhi = foldOpIntoPhi(I, PN))
2268 return NewPhi;
2269 }
2270 return nullptr;
2271}
2272
2274 // If this GEP has only 0 indices, it is the same pointer as
2275 // Src. If Src is not a trivial GEP too, don't combine
2276 // the indices.
2277 if (GEP.hasAllZeroIndices() && !Src.hasAllZeroIndices() &&
2278 !Src.hasOneUse())
2279 return false;
2280 return true;
2281}
2282
2283/// Find a constant NewC that has property:
2284/// shuffle(NewC, ShMask) = C
2285/// Returns nullptr if such a constant does not exist e.g. ShMask=<0,0> C=<1,2>
2286///
2287/// A 1-to-1 mapping is not required. Example:
2288/// ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = <poison,5,6,poison>
2290 VectorType *NewCTy) {
2291 if (isa<ScalableVectorType>(NewCTy)) {
2292 Constant *Splat = C->getSplatValue();
2293 if (!Splat)
2294 return nullptr;
2296 }
2297
2298 if (cast<FixedVectorType>(NewCTy)->getNumElements() >
2299 cast<FixedVectorType>(C->getType())->getNumElements())
2300 return nullptr;
2301
2302 unsigned NewCNumElts = cast<FixedVectorType>(NewCTy)->getNumElements();
2303 PoisonValue *PoisonScalar = PoisonValue::get(C->getType()->getScalarType());
2304 SmallVector<Constant *, 16> NewVecC(NewCNumElts, PoisonScalar);
2305 unsigned NumElts = cast<FixedVectorType>(C->getType())->getNumElements();
2306 for (unsigned I = 0; I < NumElts; ++I) {
2307 Constant *CElt = C->getAggregateElement(I);
2308 if (ShMask[I] >= 0) {
2309 assert(ShMask[I] < (int)NumElts && "Not expecting narrowing shuffle");
2310 Constant *NewCElt = NewVecC[ShMask[I]];
2311 // Bail out if:
2312 // 1. The constant vector contains a constant expression.
2313 // 2. The shuffle needs an element of the constant vector that can't
2314 // be mapped to a new constant vector.
2315 // 3. This is a widening shuffle that copies elements of V1 into the
2316 // extended elements (extending with poison is allowed).
2317 if (!CElt || (!isa<PoisonValue>(NewCElt) && NewCElt != CElt) ||
2318 I >= NewCNumElts)
2319 return nullptr;
2320 NewVecC[ShMask[I]] = CElt;
2321 }
2322 }
2323 return ConstantVector::get(NewVecC);
2324}
2325
2327 if (!isa<VectorType>(Inst.getType()))
2328 return nullptr;
2329
2330 BinaryOperator::BinaryOps Opcode = Inst.getOpcode();
2331 Value *LHS = Inst.getOperand(0), *RHS = Inst.getOperand(1);
2332 assert(cast<VectorType>(LHS->getType())->getElementCount() ==
2333 cast<VectorType>(Inst.getType())->getElementCount());
2334 assert(cast<VectorType>(RHS->getType())->getElementCount() ==
2335 cast<VectorType>(Inst.getType())->getElementCount());
2336
2337 // If both operands of the binop are vector concatenations, then perform the
2338 // narrow binop on each pair of the source operands followed by concatenation
2339 // of the results.
2340 Value *L0, *L1, *R0, *R1;
2341 ArrayRef<int> Mask;
2342 if (match(LHS, m_Shuffle(m_Value(L0), m_Value(L1), m_Mask(Mask))) &&
2343 match(RHS, m_Shuffle(m_Value(R0), m_Value(R1), m_SpecificMask(Mask))) &&
2344 LHS->hasOneUse() && RHS->hasOneUse() &&
2345 cast<ShuffleVectorInst>(LHS)->isConcat() &&
2346 cast<ShuffleVectorInst>(RHS)->isConcat()) {
2347 // This transform does not have the speculative execution constraint as
2348 // below because the shuffle is a concatenation. The new binops are
2349 // operating on exactly the same elements as the existing binop.
2350 // TODO: We could ease the mask requirement to allow different undef lanes,
2351 // but that requires an analysis of the binop-with-undef output value.
2352 Value *NewBO0 = Builder.CreateBinOp(Opcode, L0, R0);
2353 if (auto *BO = dyn_cast<BinaryOperator>(NewBO0))
2354 BO->copyIRFlags(&Inst);
2355 Value *NewBO1 = Builder.CreateBinOp(Opcode, L1, R1);
2356 if (auto *BO = dyn_cast<BinaryOperator>(NewBO1))
2357 BO->copyIRFlags(&Inst);
2358 return new ShuffleVectorInst(NewBO0, NewBO1, Mask);
2359 }
2360
2361 auto createBinOpReverse = [&](Value *X, Value *Y) {
2362 Value *V = Builder.CreateBinOp(Opcode, X, Y, Inst.getName());
2363 if (auto *BO = dyn_cast<BinaryOperator>(V))
2364 BO->copyIRFlags(&Inst);
2365 Module *M = Inst.getModule();
2367 M, Intrinsic::vector_reverse, V->getType());
2368 return CallInst::Create(F, V);
2369 };
2370
2371 // NOTE: Reverse shuffles don't require the speculative execution protection
2372 // below because they don't affect which lanes take part in the computation.
2373
2374 Value *V1, *V2;
2375 if (match(LHS, m_VecReverse(m_Value(V1)))) {
2376 // Op(rev(V1), rev(V2)) -> rev(Op(V1, V2))
2377 if (match(RHS, m_VecReverse(m_Value(V2))) &&
2378 (LHS->hasOneUse() || RHS->hasOneUse() ||
2379 (LHS == RHS && LHS->hasNUses(2))))
2380 return createBinOpReverse(V1, V2);
2381
2382 // Op(rev(V1), RHSSplat)) -> rev(Op(V1, RHSSplat))
2383 if (LHS->hasOneUse() && isSplatValue(RHS))
2384 return createBinOpReverse(V1, RHS);
2385 }
2386 // Op(LHSSplat, rev(V2)) -> rev(Op(LHSSplat, V2))
2387 else if (isSplatValue(LHS) && match(RHS, m_OneUse(m_VecReverse(m_Value(V2)))))
2388 return createBinOpReverse(LHS, V2);
2389
2390 auto createBinOpVPReverse = [&](Value *X, Value *Y, Value *EVL) {
2391 Value *V = Builder.CreateBinOp(Opcode, X, Y, Inst.getName());
2392 if (auto *BO = dyn_cast<BinaryOperator>(V))
2393 BO->copyIRFlags(&Inst);
2394
2395 ElementCount EC = cast<VectorType>(V->getType())->getElementCount();
2396 Value *AllTrueMask = Builder.CreateVectorSplat(EC, Builder.getTrue());
2397 Module *M = Inst.getModule();
2399 M, Intrinsic::experimental_vp_reverse, V->getType());
2400 return CallInst::Create(F, {V, AllTrueMask, EVL});
2401 };
2402
2403 Value *EVL;
2405 m_Value(V1), m_AllOnes(), m_Value(EVL)))) {
2406 // Op(rev(V1), rev(V2)) -> rev(Op(V1, V2))
2408 m_Value(V2), m_AllOnes(), m_Specific(EVL))) &&
2409 (LHS->hasOneUse() || RHS->hasOneUse() ||
2410 (LHS == RHS && LHS->hasNUses(2))))
2411 return createBinOpVPReverse(V1, V2, EVL);
2412
2413 // Op(rev(V1), RHSSplat)) -> rev(Op(V1, RHSSplat))
2414 if (LHS->hasOneUse() && isSplatValue(RHS))
2415 return createBinOpVPReverse(V1, RHS, EVL);
2416 }
2417 // Op(LHSSplat, rev(V2)) -> rev(Op(LHSSplat, V2))
2418 else if (isSplatValue(LHS) &&
2420 m_Value(V2), m_AllOnes(), m_Value(EVL))))
2421 return createBinOpVPReverse(LHS, V2, EVL);
2422
2423 // It may not be safe to reorder shuffles and things like div, urem, etc.
2424 // because we may trap when executing those ops on unknown vector elements.
2425 // See PR20059.
2427 return nullptr;
2428
2429 auto createBinOpShuffle = [&](Value *X, Value *Y, ArrayRef<int> M) {
2430 Value *XY = Builder.CreateBinOp(Opcode, X, Y);
2431 if (auto *BO = dyn_cast<BinaryOperator>(XY))
2432 BO->copyIRFlags(&Inst);
2433 return new ShuffleVectorInst(XY, M);
2434 };
2435
2436 // If both arguments of the binary operation are shuffles that use the same
2437 // mask and shuffle within a single vector, move the shuffle after the binop.
2438 if (match(LHS, m_Shuffle(m_Value(V1), m_Poison(), m_Mask(Mask))) &&
2439 match(RHS, m_Shuffle(m_Value(V2), m_Poison(), m_SpecificMask(Mask))) &&
2440 V1->getType() == V2->getType() &&
2441 (LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) {
2442 // Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask)
2443 return createBinOpShuffle(V1, V2, Mask);
2444 }
2445
2446 // If both arguments of a commutative binop are select-shuffles that use the
2447 // same mask with commuted operands, the shuffles are unnecessary.
2448 if (Inst.isCommutative() &&
2449 match(LHS, m_Shuffle(m_Value(V1), m_Value(V2), m_Mask(Mask))) &&
2450 match(RHS,
2451 m_Shuffle(m_Specific(V2), m_Specific(V1), m_SpecificMask(Mask)))) {
2452 auto *LShuf = cast<ShuffleVectorInst>(LHS);
2453 auto *RShuf = cast<ShuffleVectorInst>(RHS);
2454 // TODO: Allow shuffles that contain undefs in the mask?
2455 // That is legal, but it reduces undef knowledge.
2456 // TODO: Allow arbitrary shuffles by shuffling after binop?
2457 // That might be legal, but we have to deal with poison.
2458 if (LShuf->isSelect() &&
2459 !is_contained(LShuf->getShuffleMask(), PoisonMaskElem) &&
2460 RShuf->isSelect() &&
2461 !is_contained(RShuf->getShuffleMask(), PoisonMaskElem)) {
2462 // Example:
2463 // LHS = shuffle V1, V2, <0, 5, 6, 3>
2464 // RHS = shuffle V2, V1, <0, 5, 6, 3>
2465 // LHS + RHS --> (V10+V20, V21+V11, V22+V12, V13+V23) --> V1 + V2
2466 Instruction *NewBO = BinaryOperator::Create(Opcode, V1, V2);
2467 NewBO->copyIRFlags(&Inst);
2468 return NewBO;
2469 }
2470 }
2471
2472 // If one argument is a shuffle within one vector and the other is a constant,
2473 // try moving the shuffle after the binary operation. This canonicalization
2474 // intends to move shuffles closer to other shuffles and binops closer to
2475 // other binops, so they can be folded. It may also enable demanded elements
2476 // transforms.
2477 Constant *C;
2479 m_Mask(Mask))),
2480 m_ImmConstant(C)))) {
2481 assert(Inst.getType()->getScalarType() == V1->getType()->getScalarType() &&
2482 "Shuffle should not change scalar type");
2483
2484 bool ConstOp1 = isa<Constant>(RHS);
2485 if (Constant *NewC =
2487 // For fixed vectors, lanes of NewC not used by the shuffle will be poison
2488 // which will cause UB for div/rem. Mask them with a safe constant.
2489 if (isa<FixedVectorType>(V1->getType()) && Inst.isIntDivRem())
2490 NewC = getSafeVectorConstantForBinop(Opcode, NewC, ConstOp1);
2491
2492 // Op(shuffle(V1, Mask), C) -> shuffle(Op(V1, NewC), Mask)
2493 // Op(C, shuffle(V1, Mask)) -> shuffle(Op(NewC, V1), Mask)
2494 Value *NewLHS = ConstOp1 ? V1 : NewC;
2495 Value *NewRHS = ConstOp1 ? NewC : V1;
2496 return createBinOpShuffle(NewLHS, NewRHS, Mask);
2497 }
2498 }
2499
2500 // Try to reassociate to sink a splat shuffle after a binary operation.
2501 if (Inst.isAssociative() && Inst.isCommutative()) {
2502 // Canonicalize shuffle operand as LHS.
2503 if (isa<ShuffleVectorInst>(RHS))
2504 std::swap(LHS, RHS);
2505
2506 Value *X;
2507 ArrayRef<int> MaskC;
2508 int SplatIndex;
2509 Value *Y, *OtherOp;
2510 if (!match(LHS,
2511 m_OneUse(m_Shuffle(m_Value(X), m_Undef(), m_Mask(MaskC)))) ||
2512 !match(MaskC, m_SplatOrPoisonMask(SplatIndex)) ||
2513 X->getType() != Inst.getType() ||
2514 !match(RHS, m_OneUse(m_BinOp(Opcode, m_Value(Y), m_Value(OtherOp)))))
2515 return nullptr;
2516
2517 // FIXME: This may not be safe if the analysis allows undef elements. By
2518 // moving 'Y' before the splat shuffle, we are implicitly assuming
2519 // that it is not undef/poison at the splat index.
2520 if (isSplatValue(OtherOp, SplatIndex)) {
2521 std::swap(Y, OtherOp);
2522 } else if (!isSplatValue(Y, SplatIndex)) {
2523 return nullptr;
2524 }
2525
2526 // X and Y are splatted values, so perform the binary operation on those
2527 // values followed by a splat followed by the 2nd binary operation:
2528 // bo (splat X), (bo Y, OtherOp) --> bo (splat (bo X, Y)), OtherOp
2529 Value *NewBO = Builder.CreateBinOp(Opcode, X, Y);
2530 SmallVector<int, 8> NewMask(MaskC.size(), SplatIndex);
2531 Value *NewSplat = Builder.CreateShuffleVector(NewBO, NewMask);
2532 Instruction *R = BinaryOperator::Create(Opcode, NewSplat, OtherOp);
2533
2534 // Intersect FMF on both new binops. Other (poison-generating) flags are
2535 // dropped to be safe.
2536 if (isa<FPMathOperator>(R)) {
2537 R->copyFastMathFlags(&Inst);
2538 R->andIRFlags(RHS);
2539 }
2540 if (auto *NewInstBO = dyn_cast<BinaryOperator>(NewBO))
2541 NewInstBO->copyIRFlags(R);
2542 return R;
2543 }
2544
2545 return nullptr;
2546}
2547
2548/// Try to narrow the width of a binop if at least 1 operand is an extend of
2549/// of a value. This requires a potentially expensive known bits check to make
2550/// sure the narrow op does not overflow.
2551Instruction *InstCombinerImpl::narrowMathIfNoOverflow(BinaryOperator &BO) {
2552 // We need at least one extended operand.
2553 Value *Op0 = BO.getOperand(0), *Op1 = BO.getOperand(1);
2554
2555 // If this is a sub, we swap the operands since we always want an extension
2556 // on the RHS. The LHS can be an extension or a constant.
2557 if (BO.getOpcode() == Instruction::Sub)
2558 std::swap(Op0, Op1);
2559
2560 Value *X;
2561 bool IsSext = match(Op0, m_SExt(m_Value(X)));
2562 if (!IsSext && !match(Op0, m_ZExt(m_Value(X))))
2563 return nullptr;
2564
2565 // If both operands are the same extension from the same source type and we
2566 // can eliminate at least one (hasOneUse), this might work.
2567 CastInst::CastOps CastOpc = IsSext ? Instruction::SExt : Instruction::ZExt;
2568 Value *Y;
2569 if (!(match(Op1, m_ZExtOrSExt(m_Value(Y))) && X->getType() == Y->getType() &&
2570 cast<Operator>(Op1)->getOpcode() == CastOpc &&
2571 (Op0->hasOneUse() || Op1->hasOneUse()))) {
2572 // If that did not match, see if we have a suitable constant operand.
2573 // Truncating and extending must produce the same constant.
2574 Constant *WideC;
2575 if (!Op0->hasOneUse() || !match(Op1, m_Constant(WideC)))
2576 return nullptr;
2577 Constant *NarrowC = getLosslessInvCast(WideC, X->getType(), CastOpc, DL);
2578 if (!NarrowC)
2579 return nullptr;
2580 Y = NarrowC;
2581 }
2582
2583 // Swap back now that we found our operands.
2584 if (BO.getOpcode() == Instruction::Sub)
2585 std::swap(X, Y);
2586
2587 // Both operands have narrow versions. Last step: the math must not overflow
2588 // in the narrow width.
2589 if (!willNotOverflow(BO.getOpcode(), X, Y, BO, IsSext))
2590 return nullptr;
2591
2592 // bo (ext X), (ext Y) --> ext (bo X, Y)
2593 // bo (ext X), C --> ext (bo X, C')
2594 Value *NarrowBO = Builder.CreateBinOp(BO.getOpcode(), X, Y, "narrow");
2595 if (auto *NewBinOp = dyn_cast<BinaryOperator>(NarrowBO)) {
2596 if (IsSext)
2597 NewBinOp->setHasNoSignedWrap();
2598 else
2599 NewBinOp->setHasNoUnsignedWrap();
2600 }
2601 return CastInst::Create(CastOpc, NarrowBO, BO.getType());
2602}
2603
2604/// Determine nowrap flags for (gep (gep p, x), y) to (gep p, (x + y))
2605/// transform.
2610
2611/// Thread a GEP operation with constant indices through the constant true/false
2612/// arms of a select.
2614 InstCombiner::BuilderTy &Builder) {
2615 if (!GEP.hasAllConstantIndices())
2616 return nullptr;
2617
2618 Instruction *Sel;
2619 Value *Cond;
2620 Constant *TrueC, *FalseC;
2621 if (!match(GEP.getPointerOperand(), m_Instruction(Sel)) ||
2622 !match(Sel,
2623 m_Select(m_Value(Cond), m_Constant(TrueC), m_Constant(FalseC))))
2624 return nullptr;
2625
2626 // gep (select Cond, TrueC, FalseC), IndexC --> select Cond, TrueC', FalseC'
2627 // Propagate 'inbounds' and metadata from existing instructions.
2628 // Note: using IRBuilder to create the constants for efficiency.
2629 SmallVector<Value *, 4> IndexC(GEP.indices());
2630 GEPNoWrapFlags NW = GEP.getNoWrapFlags();
2631 Type *Ty = GEP.getSourceElementType();
2632 Value *NewTrueC = Builder.CreateGEP(Ty, TrueC, IndexC, "", NW);
2633 Value *NewFalseC = Builder.CreateGEP(Ty, FalseC, IndexC, "", NW);
2634 return SelectInst::Create(Cond, NewTrueC, NewFalseC, "", nullptr, Sel);
2635}
2636
2637// Canonicalization:
2638// gep T, (gep i8, base, C1), (Index + C2) into
2639// gep T, (gep i8, base, C1 + C2 * sizeof(T)), Index
2641 GEPOperator *Src,
2642 InstCombinerImpl &IC) {
2643 if (GEP.getNumIndices() != 1)
2644 return nullptr;
2645 auto &DL = IC.getDataLayout();
2646 Value *Base;
2647 const APInt *C1;
2648 if (!match(Src, m_PtrAdd(m_Value(Base), m_APInt(C1))))
2649 return nullptr;
2650 Value *VarIndex;
2651 const APInt *C2;
2652 Type *PtrTy = Src->getType()->getScalarType();
2653 unsigned IndexSizeInBits = DL.getIndexTypeSizeInBits(PtrTy);
2654 if (!match(GEP.getOperand(1), m_AddLike(m_Value(VarIndex), m_APInt(C2))))
2655 return nullptr;
2656 if (C1->getBitWidth() != IndexSizeInBits ||
2657 C2->getBitWidth() != IndexSizeInBits)
2658 return nullptr;
2659 Type *BaseType = GEP.getSourceElementType();
2661 return nullptr;
2662 APInt TypeSize(IndexSizeInBits, DL.getTypeAllocSize(BaseType));
2663 APInt NewOffset = TypeSize * *C2 + *C1;
2664 if (NewOffset.isZero() ||
2665 (Src->hasOneUse() && GEP.getOperand(1)->hasOneUse())) {
2667 if (GEP.hasNoUnsignedWrap() &&
2668 cast<GEPOperator>(Src)->hasNoUnsignedWrap() &&
2669 match(GEP.getOperand(1), m_NUWAddLike(m_Value(), m_Value()))) {
2671 if (GEP.isInBounds() && cast<GEPOperator>(Src)->isInBounds())
2672 Flags |= GEPNoWrapFlags::inBounds();
2673 }
2674
2675 Value *GEPConst =
2676 IC.Builder.CreatePtrAdd(Base, IC.Builder.getInt(NewOffset), "", Flags);
2677 return GetElementPtrInst::Create(BaseType, GEPConst, VarIndex, Flags);
2678 }
2679
2680 return nullptr;
2681}
2682
2683/// Combine constant offsets separated by variable offsets.
2684/// ptradd (ptradd (ptradd p, C1), x), C2 -> ptradd (ptradd p, x), C1+C2
2686 InstCombinerImpl &IC) {
2687 if (!GEP.hasAllConstantIndices())
2688 return nullptr;
2689
2692 auto *InnerGEP = dyn_cast<GetElementPtrInst>(GEP.getPointerOperand());
2693 while (true) {
2694 if (!InnerGEP)
2695 return nullptr;
2696
2697 NW = NW.intersectForReassociate(InnerGEP->getNoWrapFlags());
2698 if (InnerGEP->hasAllConstantIndices())
2699 break;
2700
2701 if (!InnerGEP->hasOneUse())
2702 return nullptr;
2703
2704 Skipped.push_back(InnerGEP);
2705 InnerGEP = dyn_cast<GetElementPtrInst>(InnerGEP->getPointerOperand());
2706 }
2707
2708 // The two constant offset GEPs are directly adjacent: Let normal offset
2709 // merging handle it.
2710 if (Skipped.empty())
2711 return nullptr;
2712
2713 // FIXME: This one-use check is not strictly necessary. Consider relaxing it
2714 // if profitable.
2715 if (!InnerGEP->hasOneUse())
2716 return nullptr;
2717
2718 // Don't bother with vector splats.
2719 Type *Ty = GEP.getType();
2720 if (InnerGEP->getType() != Ty)
2721 return nullptr;
2722
2723 const DataLayout &DL = IC.getDataLayout();
2724 APInt Offset(DL.getIndexTypeSizeInBits(Ty), 0);
2725 if (!GEP.accumulateConstantOffset(DL, Offset) ||
2726 !InnerGEP->accumulateConstantOffset(DL, Offset))
2727 return nullptr;
2728
2729 IC.replaceOperand(*Skipped.back(), 0, InnerGEP->getPointerOperand());
2730 for (GetElementPtrInst *SkippedGEP : Skipped)
2731 SkippedGEP->setNoWrapFlags(NW);
2732
2733 return IC.replaceInstUsesWith(
2734 GEP,
2735 IC.Builder.CreatePtrAdd(Skipped.front(), IC.Builder.getInt(Offset), "",
2736 NW.intersectForOffsetAdd(GEP.getNoWrapFlags())));
2737}
2738
2740 GEPOperator *Src) {
2741 // Combine Indices - If the source pointer to this getelementptr instruction
2742 // is a getelementptr instruction with matching element type, combine the
2743 // indices of the two getelementptr instructions into a single instruction.
2744 if (!shouldMergeGEPs(*cast<GEPOperator>(&GEP), *Src))
2745 return nullptr;
2746
2747 if (auto *I = canonicalizeGEPOfConstGEPI8(GEP, Src, *this))
2748 return I;
2749
2750 if (auto *I = combineConstantOffsets(GEP, *this))
2751 return I;
2752
2753 if (Src->getResultElementType() != GEP.getSourceElementType())
2754 return nullptr;
2755
2756 // Find out whether the last index in the source GEP is a sequential idx.
2757 bool EndsWithSequential = false;
2758 for (gep_type_iterator I = gep_type_begin(*Src), E = gep_type_end(*Src);
2759 I != E; ++I)
2760 EndsWithSequential = I.isSequential();
2761 if (!EndsWithSequential)
2762 return nullptr;
2763
2764 // Replace: gep (gep %P, long B), long A, ...
2765 // With: T = long A+B; gep %P, T, ...
2766 Value *SO1 = Src->getOperand(Src->getNumOperands() - 1);
2767 Value *GO1 = GEP.getOperand(1);
2768
2769 // If they aren't the same type, then the input hasn't been processed
2770 // by the loop above yet (which canonicalizes sequential index types to
2771 // intptr_t). Just avoid transforming this until the input has been
2772 // normalized.
2773 if (SO1->getType() != GO1->getType())
2774 return nullptr;
2775
2776 Value *Sum =
2777 simplifyAddInst(GO1, SO1, false, false, SQ.getWithInstruction(&GEP));
2778 // Only do the combine when we are sure the cost after the
2779 // merge is never more than that before the merge.
2780 if (Sum == nullptr)
2781 return nullptr;
2782
2784 Indices.append(Src->op_begin() + 1, Src->op_end() - 1);
2785 Indices.push_back(Sum);
2786 Indices.append(GEP.op_begin() + 2, GEP.op_end());
2787
2788 // Don't create GEPs with more than one non-zero index.
2789 unsigned NumNonZeroIndices = count_if(Indices, [](Value *Idx) {
2790 auto *C = dyn_cast<Constant>(Idx);
2791 return !C || !C->isNullValue();
2792 });
2793 if (NumNonZeroIndices > 1)
2794 return nullptr;
2795
2796 return replaceInstUsesWith(
2797 GEP, Builder.CreateGEP(
2798 Src->getSourceElementType(), Src->getOperand(0), Indices, "",
2800}
2801
2804 bool &DoesConsume, unsigned Depth) {
2805 static Value *const NonNull = reinterpret_cast<Value *>(uintptr_t(1));
2806 // ~(~(X)) -> X.
2807 Value *A, *B;
2808 if (match(V, m_Not(m_Value(A)))) {
2809 DoesConsume = true;
2810 return A;
2811 }
2812
2813 Constant *C;
2814 // Constants can be considered to be not'ed values.
2815 if (match(V, m_ImmConstant(C)))
2816 return ConstantExpr::getNot(C);
2817
2819 return nullptr;
2820
2821 // The rest of the cases require that we invert all uses so don't bother
2822 // doing the analysis if we know we can't use the result.
2823 if (!WillInvertAllUses)
2824 return nullptr;
2825
2826 // Compares can be inverted if all of their uses are being modified to use
2827 // the ~V.
2828 if (auto *I = dyn_cast<CmpInst>(V)) {
2829 if (Builder != nullptr)
2830 return Builder->CreateCmp(I->getInversePredicate(), I->getOperand(0),
2831 I->getOperand(1));
2832 return NonNull;
2833 }
2834
2835 // If `V` is of the form `A + B` then `-1 - V` can be folded into
2836 // `(-1 - B) - A` if we are willing to invert all of the uses.
2837 if (match(V, m_Add(m_Value(A), m_Value(B)))) {
2838 if (auto *BV = getFreelyInvertedImpl(B, B->hasOneUse(), Builder,
2839 DoesConsume, Depth))
2840 return Builder ? Builder->CreateSub(BV, A) : NonNull;
2841 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2842 DoesConsume, Depth))
2843 return Builder ? Builder->CreateSub(AV, B) : NonNull;
2844 return nullptr;
2845 }
2846
2847 // If `V` is of the form `A ^ ~B` then `~(A ^ ~B)` can be folded
2848 // into `A ^ B` if we are willing to invert all of the uses.
2849 if (match(V, m_Xor(m_Value(A), m_Value(B)))) {
2850 if (auto *BV = getFreelyInvertedImpl(B, B->hasOneUse(), Builder,
2851 DoesConsume, Depth))
2852 return Builder ? Builder->CreateXor(A, BV) : NonNull;
2853 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2854 DoesConsume, Depth))
2855 return Builder ? Builder->CreateXor(AV, B) : NonNull;
2856 return nullptr;
2857 }
2858
2859 // If `V` is of the form `B - A` then `-1 - V` can be folded into
2860 // `A + (-1 - B)` if we are willing to invert all of the uses.
2861 if (match(V, m_Sub(m_Value(A), m_Value(B)))) {
2862 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2863 DoesConsume, Depth))
2864 return Builder ? Builder->CreateAdd(AV, B) : NonNull;
2865 return nullptr;
2866 }
2867
2868 // If `V` is of the form `(~A) s>> B` then `~((~A) s>> B)` can be folded
2869 // into `A s>> B` if we are willing to invert all of the uses.
2870 if (match(V, m_AShr(m_Value(A), m_Value(B)))) {
2871 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2872 DoesConsume, Depth))
2873 return Builder ? Builder->CreateAShr(AV, B) : NonNull;
2874 return nullptr;
2875 }
2876
2877 Value *Cond;
2878 // LogicOps are special in that we canonicalize them at the cost of an
2879 // instruction.
2880 bool IsSelect = match(V, m_Select(m_Value(Cond), m_Value(A), m_Value(B))) &&
2882 // Selects/min/max with invertible operands are freely invertible
2883 if (IsSelect || match(V, m_MaxOrMin(m_Value(A), m_Value(B)))) {
2884 bool LocalDoesConsume = DoesConsume;
2885 if (!getFreelyInvertedImpl(B, B->hasOneUse(), /*Builder*/ nullptr,
2886 LocalDoesConsume, Depth))
2887 return nullptr;
2888 if (Value *NotA = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2889 LocalDoesConsume, Depth)) {
2890 DoesConsume = LocalDoesConsume;
2891 if (Builder != nullptr) {
2892 Value *NotB = getFreelyInvertedImpl(B, B->hasOneUse(), Builder,
2893 DoesConsume, Depth);
2894 assert(NotB != nullptr &&
2895 "Unable to build inverted value for known freely invertable op");
2896 if (auto *II = dyn_cast<IntrinsicInst>(V))
2897 return Builder->CreateBinaryIntrinsic(
2898 getInverseMinMaxIntrinsic(II->getIntrinsicID()), NotA, NotB);
2899 return Builder->CreateSelect(Cond, NotA, NotB);
2900 }
2901 return NonNull;
2902 }
2903 }
2904
2905 if (PHINode *PN = dyn_cast<PHINode>(V)) {
2906 bool LocalDoesConsume = DoesConsume;
2908 for (Use &U : PN->operands()) {
2909 BasicBlock *IncomingBlock = PN->getIncomingBlock(U);
2910 Value *NewIncomingVal = getFreelyInvertedImpl(
2911 U.get(), /*WillInvertAllUses=*/false,
2912 /*Builder=*/nullptr, LocalDoesConsume, MaxAnalysisRecursionDepth - 1);
2913 if (NewIncomingVal == nullptr)
2914 return nullptr;
2915 // Make sure that we can safely erase the original PHI node.
2916 if (NewIncomingVal == V)
2917 return nullptr;
2918 if (Builder != nullptr)
2919 IncomingValues.emplace_back(NewIncomingVal, IncomingBlock);
2920 }
2921
2922 DoesConsume = LocalDoesConsume;
2923 if (Builder != nullptr) {
2925 Builder->SetInsertPoint(PN);
2926 PHINode *NewPN =
2927 Builder->CreatePHI(PN->getType(), PN->getNumIncomingValues());
2928 for (auto [Val, Pred] : IncomingValues)
2929 NewPN->addIncoming(Val, Pred);
2930 return NewPN;
2931 }
2932 return NonNull;
2933 }
2934
2935 if (match(V, m_SExtLike(m_Value(A)))) {
2936 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2937 DoesConsume, Depth))
2938 return Builder ? Builder->CreateSExt(AV, V->getType()) : NonNull;
2939 return nullptr;
2940 }
2941
2942 if (match(V, m_Trunc(m_Value(A)))) {
2943 if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2944 DoesConsume, Depth))
2945 return Builder ? Builder->CreateTrunc(AV, V->getType()) : NonNull;
2946 return nullptr;
2947 }
2948
2949 // De Morgan's Laws:
2950 // (~(A | B)) -> (~A & ~B)
2951 // (~(A & B)) -> (~A | ~B)
2952 auto TryInvertAndOrUsingDeMorgan = [&](Instruction::BinaryOps Opcode,
2953 bool IsLogical, Value *A,
2954 Value *B) -> Value * {
2955 bool LocalDoesConsume = DoesConsume;
2956 if (!getFreelyInvertedImpl(B, B->hasOneUse(), /*Builder=*/nullptr,
2957 LocalDoesConsume, Depth))
2958 return nullptr;
2959 if (auto *NotA = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
2960 LocalDoesConsume, Depth)) {
2961 auto *NotB = getFreelyInvertedImpl(B, B->hasOneUse(), Builder,
2962 LocalDoesConsume, Depth);
2963 DoesConsume = LocalDoesConsume;
2964 if (IsLogical)
2965 return Builder ? Builder->CreateLogicalOp(Opcode, NotA, NotB) : NonNull;
2966 return Builder ? Builder->CreateBinOp(Opcode, NotA, NotB) : NonNull;
2967 }
2968
2969 return nullptr;
2970 };
2971
2972 if (match(V, m_Or(m_Value(A), m_Value(B))))
2973 return TryInvertAndOrUsingDeMorgan(Instruction::And, /*IsLogical=*/false, A,
2974 B);
2975
2976 if (match(V, m_And(m_Value(A), m_Value(B))))
2977 return TryInvertAndOrUsingDeMorgan(Instruction::Or, /*IsLogical=*/false, A,
2978 B);
2979
2980 if (match(V, m_LogicalOr(m_Value(A), m_Value(B))))
2981 return TryInvertAndOrUsingDeMorgan(Instruction::And, /*IsLogical=*/true, A,
2982 B);
2983
2984 if (match(V, m_LogicalAnd(m_Value(A), m_Value(B))))
2985 return TryInvertAndOrUsingDeMorgan(Instruction::Or, /*IsLogical=*/true, A,
2986 B);
2987
2988 return nullptr;
2989}
2990
2991/// Return true if we should canonicalize the gep to an i8 ptradd.
2993 Value *PtrOp = GEP.getOperand(0);
2994 Type *GEPEltType = GEP.getSourceElementType();
2995 if (GEPEltType->isIntegerTy(8))
2996 return false;
2997
2998 // Canonicalize scalable GEPs to an explicit offset using the llvm.vscale
2999 // intrinsic. This has better support in BasicAA.
3000 if (GEPEltType->isScalableTy())
3001 return true;
3002
3003 // gep i32 p, mul(O, C) -> gep i8, p, mul(O, C*4) to fold the two multiplies
3004 // together.
3005 if (GEP.getNumIndices() == 1 &&
3006 match(GEP.getOperand(1),
3008 m_Shl(m_Value(), m_ConstantInt())))))
3009 return true;
3010
3011 // gep (gep %p, C1), %x, C2 is expanded so the two constants can
3012 // possibly be merged together.
3013 auto PtrOpGep = dyn_cast<GEPOperator>(PtrOp);
3014 return PtrOpGep && PtrOpGep->hasAllConstantIndices() &&
3015 any_of(GEP.indices(), [](Value *V) {
3016 const APInt *C;
3017 return match(V, m_APInt(C)) && !C->isZero();
3018 });
3019}
3020
3022 IRBuilderBase &Builder) {
3023 auto *Op1 = dyn_cast<GetElementPtrInst>(PN->getOperand(0));
3024 if (!Op1)
3025 return nullptr;
3026
3027 // Don't fold a GEP into itself through a PHI node. This can only happen
3028 // through the back-edge of a loop. Folding a GEP into itself means that
3029 // the value of the previous iteration needs to be stored in the meantime,
3030 // thus requiring an additional register variable to be live, but not
3031 // actually achieving anything (the GEP still needs to be executed once per
3032 // loop iteration).
3033 if (Op1 == &GEP)
3034 return nullptr;
3035 GEPNoWrapFlags NW = Op1->getNoWrapFlags();
3036
3037 int DI = -1;
3038
3039 for (auto I = PN->op_begin()+1, E = PN->op_end(); I !=E; ++I) {
3040 auto *Op2 = dyn_cast<GetElementPtrInst>(*I);
3041 if (!Op2 || Op1->getNumOperands() != Op2->getNumOperands() ||
3042 Op1->getSourceElementType() != Op2->getSourceElementType())
3043 return nullptr;
3044
3045 // As for Op1 above, don't try to fold a GEP into itself.
3046 if (Op2 == &GEP)
3047 return nullptr;
3048
3049 // Keep track of the type as we walk the GEP.
3050 Type *CurTy = nullptr;
3051
3052 for (unsigned J = 0, F = Op1->getNumOperands(); J != F; ++J) {
3053 if (Op1->getOperand(J)->getType() != Op2->getOperand(J)->getType())
3054 return nullptr;
3055
3056 if (Op1->getOperand(J) != Op2->getOperand(J)) {
3057 if (DI == -1) {
3058 // We have not seen any differences yet in the GEPs feeding the
3059 // PHI yet, so we record this one if it is allowed to be a
3060 // variable.
3061
3062 // The first two arguments can vary for any GEP, the rest have to be
3063 // static for struct slots
3064 if (J > 1) {
3065 assert(CurTy && "No current type?");
3066 if (CurTy->isStructTy())
3067 return nullptr;
3068 }
3069
3070 DI = J;
3071 } else {
3072 // The GEP is different by more than one input. While this could be
3073 // extended to support GEPs that vary by more than one variable it
3074 // doesn't make sense since it greatly increases the complexity and
3075 // would result in an R+R+R addressing mode which no backend
3076 // directly supports and would need to be broken into several
3077 // simpler instructions anyway.
3078 return nullptr;
3079 }
3080 }
3081
3082 // Sink down a layer of the type for the next iteration.
3083 if (J > 0) {
3084 if (J == 1) {
3085 CurTy = Op1->getSourceElementType();
3086 } else {
3087 CurTy =
3088 GetElementPtrInst::getTypeAtIndex(CurTy, Op1->getOperand(J));
3089 }
3090 }
3091 }
3092
3093 NW &= Op2->getNoWrapFlags();
3094 }
3095
3096 // If not all GEPs are identical we'll have to create a new PHI node.
3097 // Check that the old PHI node has only one use so that it will get
3098 // removed.
3099 if (DI != -1 && !PN->hasOneUse())
3100 return nullptr;
3101
3102 auto *NewGEP = cast<GetElementPtrInst>(Op1->clone());
3103 NewGEP->setNoWrapFlags(NW);
3104
3105 if (DI == -1) {
3106 // All the GEPs feeding the PHI are identical. Clone one down into our
3107 // BB so that it can be merged with the current GEP.
3108 } else {
3109 // All the GEPs feeding the PHI differ at a single offset. Clone a GEP
3110 // into the current block so it can be merged, and create a new PHI to
3111 // set that index.
3112 PHINode *NewPN;
3113 {
3114 IRBuilderBase::InsertPointGuard Guard(Builder);
3115 Builder.SetInsertPoint(PN);
3116 NewPN = Builder.CreatePHI(Op1->getOperand(DI)->getType(),
3117 PN->getNumOperands());
3118 }
3119
3120 for (auto &I : PN->operands())
3121 NewPN->addIncoming(cast<GEPOperator>(I)->getOperand(DI),
3122 PN->getIncomingBlock(I));
3123
3124 NewGEP->setOperand(DI, NewPN);
3125 }
3126
3127 NewGEP->insertBefore(*GEP.getParent(), GEP.getParent()->getFirstInsertionPt());
3128 return NewGEP;
3129}
3130
3132 Value *PtrOp = GEP.getOperand(0);
3133 SmallVector<Value *, 8> Indices(GEP.indices());
3134 Type *GEPType = GEP.getType();
3135 Type *GEPEltType = GEP.getSourceElementType();
3136 if (Value *V =
3137 simplifyGEPInst(GEPEltType, PtrOp, Indices, GEP.getNoWrapFlags(),
3138 SQ.getWithInstruction(&GEP)))
3139 return replaceInstUsesWith(GEP, V);
3140
3141 // For vector geps, use the generic demanded vector support.
3142 // Skip if GEP return type is scalable. The number of elements is unknown at
3143 // compile-time.
3144 if (auto *GEPFVTy = dyn_cast<FixedVectorType>(GEPType)) {
3145 auto VWidth = GEPFVTy->getNumElements();
3146 APInt PoisonElts(VWidth, 0);
3147 APInt AllOnesEltMask(APInt::getAllOnes(VWidth));
3148 if (Value *V = SimplifyDemandedVectorElts(&GEP, AllOnesEltMask,
3149 PoisonElts)) {
3150 if (V != &GEP)
3151 return replaceInstUsesWith(GEP, V);
3152 return &GEP;
3153 }
3154 }
3155
3156 // Eliminate unneeded casts for indices, and replace indices which displace
3157 // by multiples of a zero size type with zero.
3158 bool MadeChange = false;
3159
3160 // Index width may not be the same width as pointer width.
3161 // Data layout chooses the right type based on supported integer types.
3162 Type *NewScalarIndexTy =
3163 DL.getIndexType(GEP.getPointerOperandType()->getScalarType());
3164
3166 for (User::op_iterator I = GEP.op_begin() + 1, E = GEP.op_end(); I != E;
3167 ++I, ++GTI) {
3168 // Skip indices into struct types.
3169 if (GTI.isStruct())
3170 continue;
3171
3172 Type *IndexTy = (*I)->getType();
3173 Type *NewIndexType =
3174 IndexTy->isVectorTy()
3175 ? VectorType::get(NewScalarIndexTy,
3176 cast<VectorType>(IndexTy)->getElementCount())
3177 : NewScalarIndexTy;
3178
3179 // If the element type has zero size then any index over it is equivalent
3180 // to an index of zero, so replace it with zero if it is not zero already.
3181 Type *EltTy = GTI.getIndexedType();
3182 if (EltTy->isSized() && DL.getTypeAllocSize(EltTy).isZero())
3183 if (!isa<Constant>(*I) || !match(I->get(), m_Zero())) {
3184 *I = Constant::getNullValue(NewIndexType);
3185 MadeChange = true;
3186 }
3187
3188 if (IndexTy != NewIndexType) {
3189 // If we are using a wider index than needed for this platform, shrink
3190 // it to what we need. If narrower, sign-extend it to what we need.
3191 // This explicit cast can make subsequent optimizations more obvious.
3192 if (IndexTy->getScalarSizeInBits() <
3193 NewIndexType->getScalarSizeInBits()) {
3194 if (GEP.hasNoUnsignedWrap() && GEP.hasNoUnsignedSignedWrap())
3195 *I = Builder.CreateZExt(*I, NewIndexType, "", /*IsNonNeg=*/true);
3196 else
3197 *I = Builder.CreateSExt(*I, NewIndexType);
3198 } else {
3199 *I = Builder.CreateTrunc(*I, NewIndexType, "", GEP.hasNoUnsignedWrap(),
3200 GEP.hasNoUnsignedSignedWrap());
3201 }
3202 MadeChange = true;
3203 }
3204 }
3205 if (MadeChange)
3206 return &GEP;
3207
3208 // Canonicalize constant GEPs to i8 type.
3209 if (!GEPEltType->isIntegerTy(8) && GEP.hasAllConstantIndices()) {
3210 APInt Offset(DL.getIndexTypeSizeInBits(GEPType), 0);
3211 if (GEP.accumulateConstantOffset(DL, Offset))
3212 return replaceInstUsesWith(
3213 GEP, Builder.CreatePtrAdd(PtrOp, Builder.getInt(Offset), "",
3214 GEP.getNoWrapFlags()));
3215 }
3216
3218 Value *Offset = EmitGEPOffset(cast<GEPOperator>(&GEP));
3219 Value *NewGEP =
3220 Builder.CreatePtrAdd(PtrOp, Offset, "", GEP.getNoWrapFlags());
3221 return replaceInstUsesWith(GEP, NewGEP);
3222 }
3223
3224 // Strip trailing zero indices.
3225 auto *LastIdx = dyn_cast<Constant>(Indices.back());
3226 if (LastIdx && LastIdx->isNullValue() && !LastIdx->getType()->isVectorTy()) {
3227 return replaceInstUsesWith(
3228 GEP, Builder.CreateGEP(GEP.getSourceElementType(), PtrOp,
3229 drop_end(Indices), "", GEP.getNoWrapFlags()));
3230 }
3231
3232 // Strip leading zero indices.
3233 auto *FirstIdx = dyn_cast<Constant>(Indices.front());
3234 if (FirstIdx && FirstIdx->isNullValue() &&
3235 !FirstIdx->getType()->isVectorTy()) {
3237 ++GTI;
3238 if (!GTI.isStruct())
3239 return replaceInstUsesWith(GEP, Builder.CreateGEP(GTI.getIndexedType(),
3240 GEP.getPointerOperand(),
3241 drop_begin(Indices), "",
3242 GEP.getNoWrapFlags()));
3243 }
3244
3245 // Scalarize vector operands; prefer splat-of-gep.as canonical form.
3246 // Note that this looses information about undef lanes; we run it after
3247 // demanded bits to partially mitigate that loss.
3248 if (GEPType->isVectorTy() && llvm::any_of(GEP.operands(), [](Value *Op) {
3249 return Op->getType()->isVectorTy() && getSplatValue(Op);
3250 })) {
3251 SmallVector<Value *> NewOps;
3252 for (auto &Op : GEP.operands()) {
3253 if (Op->getType()->isVectorTy())
3254 if (Value *Scalar = getSplatValue(Op)) {
3255 NewOps.push_back(Scalar);
3256 continue;
3257 }
3258 NewOps.push_back(Op);
3259 }
3260
3261 Value *Res = Builder.CreateGEP(GEP.getSourceElementType(), NewOps[0],
3262 ArrayRef(NewOps).drop_front(), GEP.getName(),
3263 GEP.getNoWrapFlags());
3264 if (!Res->getType()->isVectorTy()) {
3265 ElementCount EC = cast<VectorType>(GEPType)->getElementCount();
3266 Res = Builder.CreateVectorSplat(EC, Res);
3267 }
3268 return replaceInstUsesWith(GEP, Res);
3269 }
3270
3271 bool SeenNonZeroIndex = false;
3272 for (auto [IdxNum, Idx] : enumerate(Indices)) {
3273 auto *C = dyn_cast<Constant>(Idx);
3274 if (C && C->isNullValue())
3275 continue;
3276
3277 if (!SeenNonZeroIndex) {
3278 SeenNonZeroIndex = true;
3279 continue;
3280 }
3281
3282 // GEP has multiple non-zero indices: Split it.
3283 ArrayRef<Value *> FrontIndices = ArrayRef(Indices).take_front(IdxNum);
3284 Value *FrontGEP =
3285 Builder.CreateGEP(GEPEltType, PtrOp, FrontIndices,
3286 GEP.getName() + ".split", GEP.getNoWrapFlags());
3287
3288 SmallVector<Value *> BackIndices;
3289 BackIndices.push_back(Constant::getNullValue(NewScalarIndexTy));
3290 append_range(BackIndices, drop_begin(Indices, IdxNum));
3292 GetElementPtrInst::getIndexedType(GEPEltType, FrontIndices), FrontGEP,
3293 BackIndices, GEP.getNoWrapFlags());
3294 }
3295
3296 // Check to see if the inputs to the PHI node are getelementptr instructions.
3297 if (auto *PN = dyn_cast<PHINode>(PtrOp)) {
3298 if (Value *NewPtrOp = foldGEPOfPhi(GEP, PN, Builder))
3299 return replaceOperand(GEP, 0, NewPtrOp);
3300 }
3301
3302 if (auto *Src = dyn_cast<GEPOperator>(PtrOp))
3303 if (Instruction *I = visitGEPOfGEP(GEP, Src))
3304 return I;
3305
3306 if (GEP.getNumIndices() == 1) {
3307 unsigned AS = GEP.getPointerAddressSpace();
3308 if (GEP.getOperand(1)->getType()->getScalarSizeInBits() ==
3309 DL.getIndexSizeInBits(AS)) {
3310 uint64_t TyAllocSize = DL.getTypeAllocSize(GEPEltType).getFixedValue();
3311
3312 if (TyAllocSize == 1) {
3313 // Canonicalize (gep i8* X, (ptrtoint Y)-(ptrtoint X)) to (bitcast Y),
3314 // but only if the result pointer is only used as if it were an integer,
3315 // or both point to the same underlying object (otherwise provenance is
3316 // not necessarily retained).
3317 Value *X = GEP.getPointerOperand();
3318 Value *Y;
3319 if (match(GEP.getOperand(1),
3321 GEPType == Y->getType()) {
3322 bool HasSameUnderlyingObject =
3324 bool Changed = false;
3325 GEP.replaceUsesWithIf(Y, [&](Use &U) {
3326 bool ShouldReplace = HasSameUnderlyingObject ||
3327 isa<ICmpInst>(U.getUser()) ||
3328 isa<PtrToIntInst>(U.getUser());
3329 Changed |= ShouldReplace;
3330 return ShouldReplace;
3331 });
3332 return Changed ? &GEP : nullptr;
3333 }
3334 } else if (auto *ExactIns =
3335 dyn_cast<PossiblyExactOperator>(GEP.getOperand(1))) {
3336 // Canonicalize (gep T* X, V / sizeof(T)) to (gep i8* X, V)
3337 Value *V;
3338 if (ExactIns->isExact()) {
3339 if ((has_single_bit(TyAllocSize) &&
3340 match(GEP.getOperand(1),
3341 m_Shr(m_Value(V),
3342 m_SpecificInt(countr_zero(TyAllocSize))))) ||
3343 match(GEP.getOperand(1),
3344 m_IDiv(m_Value(V), m_SpecificInt(TyAllocSize)))) {
3345 return GetElementPtrInst::Create(Builder.getInt8Ty(),
3346 GEP.getPointerOperand(), V,
3347 GEP.getNoWrapFlags());
3348 }
3349 }
3350 if (ExactIns->isExact() && ExactIns->hasOneUse()) {
3351 // Try to canonicalize non-i8 element type to i8 if the index is an
3352 // exact instruction. If the index is an exact instruction (div/shr)
3353 // with a constant RHS, we can fold the non-i8 element scale into the
3354 // div/shr (similiar to the mul case, just inverted).
3355 const APInt *C;
3356 std::optional<APInt> NewC;
3357 if (has_single_bit(TyAllocSize) &&
3358 match(ExactIns, m_Shr(m_Value(V), m_APInt(C))) &&
3359 C->uge(countr_zero(TyAllocSize)))
3360 NewC = *C - countr_zero(TyAllocSize);
3361 else if (match(ExactIns, m_UDiv(m_Value(V), m_APInt(C)))) {
3362 APInt Quot;
3363 uint64_t Rem;
3364 APInt::udivrem(*C, TyAllocSize, Quot, Rem);
3365 if (Rem == 0)
3366 NewC = Quot;
3367 } else if (match(ExactIns, m_SDiv(m_Value(V), m_APInt(C)))) {
3368 APInt Quot;
3369 int64_t Rem;
3370 APInt::sdivrem(*C, TyAllocSize, Quot, Rem);
3371 // For sdiv we need to make sure we arent creating INT_MIN / -1.
3372 if (!Quot.isAllOnes() && Rem == 0)
3373 NewC = Quot;
3374 }
3375
3376 if (NewC.has_value()) {
3377 Value *NewOp = Builder.CreateBinOp(
3378 static_cast<Instruction::BinaryOps>(ExactIns->getOpcode()), V,
3379 ConstantInt::get(V->getType(), *NewC));
3380 cast<BinaryOperator>(NewOp)->setIsExact();
3381 return GetElementPtrInst::Create(Builder.getInt8Ty(),
3382 GEP.getPointerOperand(), NewOp,
3383 GEP.getNoWrapFlags());
3384 }
3385 }
3386 }
3387 }
3388 }
3389 // We do not handle pointer-vector geps here.
3390 if (GEPType->isVectorTy())
3391 return nullptr;
3392
3393 if (!GEP.isInBounds()) {
3394 unsigned IdxWidth =
3395 DL.getIndexSizeInBits(PtrOp->getType()->getPointerAddressSpace());
3396 APInt BasePtrOffset(IdxWidth, 0);
3397 Value *UnderlyingPtrOp =
3398 PtrOp->stripAndAccumulateInBoundsConstantOffsets(DL, BasePtrOffset);
3399 bool CanBeNull, CanBeFreed;
3400 uint64_t DerefBytes = UnderlyingPtrOp->getPointerDereferenceableBytes(
3401 DL, CanBeNull, CanBeFreed);
3402 if (!CanBeNull && !CanBeFreed && DerefBytes != 0) {
3403 if (GEP.accumulateConstantOffset(DL, BasePtrOffset) &&
3404 BasePtrOffset.isNonNegative()) {
3405 APInt AllocSize(IdxWidth, DerefBytes);
3406 if (BasePtrOffset.ule(AllocSize)) {
3408 GEP.getSourceElementType(), PtrOp, Indices, GEP.getName());
3409 }
3410 }
3411 }
3412 }
3413
3414 // nusw + nneg -> nuw
3415 if (GEP.hasNoUnsignedSignedWrap() && !GEP.hasNoUnsignedWrap() &&
3416 all_of(GEP.indices(), [&](Value *Idx) {
3417 return isKnownNonNegative(Idx, SQ.getWithInstruction(&GEP));
3418 })) {
3419 GEP.setNoWrapFlags(GEP.getNoWrapFlags() | GEPNoWrapFlags::noUnsignedWrap());
3420 return &GEP;
3421 }
3422
3423 // These rewrites are trying to preserve inbounds/nuw attributes. So we want
3424 // to do this after having tried to derive "nuw" above.
3425 if (GEP.getNumIndices() == 1) {
3426 // Given (gep p, x+y) we want to determine the common nowrap flags for both
3427 // geps if transforming into (gep (gep p, x), y).
3428 auto GetPreservedNoWrapFlags = [&](bool AddIsNUW) {
3429 // We can preserve both "inbounds nuw", "nusw nuw" and "nuw" if we know
3430 // that x + y does not have unsigned wrap.
3431 if (GEP.hasNoUnsignedWrap() && AddIsNUW)
3432 return GEP.getNoWrapFlags();
3433 return GEPNoWrapFlags::none();
3434 };
3435
3436 // Try to replace ADD + GEP with GEP + GEP.
3437 Value *Idx1, *Idx2;
3438 if (match(GEP.getOperand(1),
3439 m_OneUse(m_AddLike(m_Value(Idx1), m_Value(Idx2))))) {
3440 // %idx = add i64 %idx1, %idx2
3441 // %gep = getelementptr i32, ptr %ptr, i64 %idx
3442 // as:
3443 // %newptr = getelementptr i32, ptr %ptr, i64 %idx1
3444 // %newgep = getelementptr i32, ptr %newptr, i64 %idx2
3445 bool NUW = match(GEP.getOperand(1), m_NUWAddLike(m_Value(), m_Value()));
3446 GEPNoWrapFlags NWFlags = GetPreservedNoWrapFlags(NUW);
3447 auto *NewPtr =
3448 Builder.CreateGEP(GEP.getSourceElementType(), GEP.getPointerOperand(),
3449 Idx1, "", NWFlags);
3450 return replaceInstUsesWith(GEP,
3451 Builder.CreateGEP(GEP.getSourceElementType(),
3452 NewPtr, Idx2, "", NWFlags));
3453 }
3454 ConstantInt *C;
3455 if (match(GEP.getOperand(1), m_OneUse(m_SExtLike(m_OneUse(m_NSWAddLike(
3456 m_Value(Idx1), m_ConstantInt(C))))))) {
3457 // %add = add nsw i32 %idx1, idx2
3458 // %sidx = sext i32 %add to i64
3459 // %gep = getelementptr i32, ptr %ptr, i64 %sidx
3460 // as:
3461 // %newptr = getelementptr i32, ptr %ptr, i32 %idx1
3462 // %newgep = getelementptr i32, ptr %newptr, i32 idx2
3463 bool NUW = match(GEP.getOperand(1),
3465 GEPNoWrapFlags NWFlags = GetPreservedNoWrapFlags(NUW);
3466 auto *NewPtr = Builder.CreateGEP(
3467 GEP.getSourceElementType(), GEP.getPointerOperand(),
3468 Builder.CreateSExt(Idx1, GEP.getOperand(1)->getType()), "", NWFlags);
3469 return replaceInstUsesWith(
3470 GEP,
3471 Builder.CreateGEP(GEP.getSourceElementType(), NewPtr,
3472 Builder.CreateSExt(C, GEP.getOperand(1)->getType()),
3473 "", NWFlags));
3474 }
3475 }
3476
3478 return R;
3479
3480 return nullptr;
3481}
3482
3484 Instruction *AI) {
3486 return true;
3487 if (auto *LI = dyn_cast<LoadInst>(V))
3488 return isa<GlobalVariable>(LI->getPointerOperand());
3489 // Two distinct allocations will never be equal.
3490 return isAllocLikeFn(V, &TLI) && V != AI;
3491}
3492
3493/// Given a call CB which uses an address UsedV, return true if we can prove the
3494/// call's only possible effect is storing to V.
3495static bool isRemovableWrite(CallBase &CB, Value *UsedV,
3496 const TargetLibraryInfo &TLI) {
3497 if (!CB.use_empty())
3498 // TODO: add recursion if returned attribute is present
3499 return false;
3500
3501 if (CB.isTerminator())
3502 // TODO: remove implementation restriction
3503 return false;
3504
3505 if (!CB.willReturn() || !CB.doesNotThrow())
3506 return false;
3507
3508 // If the only possible side effect of the call is writing to the alloca,
3509 // and the result isn't used, we can safely remove any reads implied by the
3510 // call including those which might read the alloca itself.
3511 std::optional<MemoryLocation> Dest = MemoryLocation::getForDest(&CB, TLI);
3512 return Dest && Dest->Ptr == UsedV;
3513}
3514
3515static std::optional<ModRefInfo>
3517 const TargetLibraryInfo &TLI, bool KnowInit) {
3519 const std::optional<StringRef> Family = getAllocationFamily(AI, &TLI);
3520 Worklist.push_back(AI);
3522
3523 do {
3524 Instruction *PI = Worklist.pop_back_val();
3525 for (User *U : PI->users()) {
3527 switch (I->getOpcode()) {
3528 default:
3529 // Give up the moment we see something we can't handle.
3530 return std::nullopt;
3531
3532 case Instruction::AddrSpaceCast:
3533 case Instruction::BitCast:
3534 case Instruction::GetElementPtr:
3535 Users.emplace_back(I);
3536 Worklist.push_back(I);
3537 continue;
3538
3539 case Instruction::ICmp: {
3540 ICmpInst *ICI = cast<ICmpInst>(I);
3541 // We can fold eq/ne comparisons with null to false/true, respectively.
3542 // We also fold comparisons in some conditions provided the alloc has
3543 // not escaped (see isNeverEqualToUnescapedAlloc).
3544 if (!ICI->isEquality())
3545 return std::nullopt;
3546 unsigned OtherIndex = (ICI->getOperand(0) == PI) ? 1 : 0;
3547 if (!isNeverEqualToUnescapedAlloc(ICI->getOperand(OtherIndex), TLI, AI))
3548 return std::nullopt;
3549
3550 // Do not fold compares to aligned_alloc calls, as they may have to
3551 // return null in case the required alignment cannot be satisfied,
3552 // unless we can prove that both alignment and size are valid.
3553 auto AlignmentAndSizeKnownValid = [](CallBase *CB) {
3554 // Check if alignment and size of a call to aligned_alloc is valid,
3555 // that is alignment is a power-of-2 and the size is a multiple of the
3556 // alignment.
3557 const APInt *Alignment;
3558 const APInt *Size;
3559 return match(CB->getArgOperand(0), m_APInt(Alignment)) &&
3560 match(CB->getArgOperand(1), m_APInt(Size)) &&
3561 Alignment->isPowerOf2() && Size->urem(*Alignment).isZero();
3562 };
3563 auto *CB = dyn_cast<CallBase>(AI);
3564 LibFunc TheLibFunc;
3565 if (CB && TLI.getLibFunc(*CB->getCalledFunction(), TheLibFunc) &&
3566 TLI.has(TheLibFunc) && TheLibFunc == LibFunc_aligned_alloc &&
3567 !AlignmentAndSizeKnownValid(CB))
3568 return std::nullopt;
3569 Users.emplace_back(I);
3570 continue;
3571 }
3572
3573 case Instruction::Call:
3574 // Ignore no-op and store intrinsics.
3576 switch (II->getIntrinsicID()) {
3577 default:
3578 return std::nullopt;
3579
3580 case Intrinsic::memmove:
3581 case Intrinsic::memcpy:
3582 case Intrinsic::memset: {
3584 if (MI->isVolatile())
3585 return std::nullopt;
3586 // Note: this could also be ModRef, but we can still interpret that
3587 // as just Mod in that case.
3588 ModRefInfo NewAccess =
3589 MI->getRawDest() == PI ? ModRefInfo::Mod : ModRefInfo::Ref;
3590 if ((Access & ~NewAccess) != ModRefInfo::NoModRef)
3591 return std::nullopt;
3592 Access |= NewAccess;
3593 [[fallthrough]];
3594 }
3595 case Intrinsic::assume:
3596 case Intrinsic::invariant_start:
3597 case Intrinsic::invariant_end:
3598 case Intrinsic::lifetime_start:
3599 case Intrinsic::lifetime_end:
3600 case Intrinsic::objectsize:
3601 Users.emplace_back(I);
3602 continue;
3603 case Intrinsic::launder_invariant_group:
3604 case Intrinsic::strip_invariant_group:
3605 Users.emplace_back(I);
3606 Worklist.push_back(I);
3607 continue;
3608 }
3609 }
3610
3611 if (Family && getFreedOperand(cast<CallBase>(I), &TLI) == PI &&
3612 getAllocationFamily(I, &TLI) == Family) {
3613 Users.emplace_back(I);
3614 continue;
3615 }
3616
3617 if (Family && getReallocatedOperand(cast<CallBase>(I)) == PI &&
3618 getAllocationFamily(I, &TLI) == Family) {
3619 Users.emplace_back(I);
3620 Worklist.push_back(I);
3621 continue;
3622 }
3623
3624 if (!isRefSet(Access) &&
3625 isRemovableWrite(*cast<CallBase>(I), PI, TLI)) {
3627 Users.emplace_back(I);
3628 continue;
3629 }
3630
3631 return std::nullopt;
3632
3633 case Instruction::Store: {
3635 if (SI->isVolatile() || SI->getPointerOperand() != PI)
3636 return std::nullopt;
3637 if (isRefSet(Access))
3638 return std::nullopt;
3640 Users.emplace_back(I);
3641 continue;
3642 }
3643
3644 case Instruction::Load: {
3645 LoadInst *LI = cast<LoadInst>(I);
3646 if (LI->isVolatile() || LI->getPointerOperand() != PI)
3647 return std::nullopt;
3648 if (isModSet(Access))
3649 return std::nullopt;
3651 Users.emplace_back(I);
3652 continue;
3653 }
3654 }
3655 llvm_unreachable("missing a return?");
3656 }
3657 } while (!Worklist.empty());
3658
3660 return Access;
3661}
3662
3665
3666 // If we have a malloc call which is only used in any amount of comparisons to
3667 // null and free calls, delete the calls and replace the comparisons with true
3668 // or false as appropriate.
3669
3670 // This is based on the principle that we can substitute our own allocation
3671 // function (which will never return null) rather than knowledge of the
3672 // specific function being called. In some sense this can change the permitted
3673 // outputs of a program (when we convert a malloc to an alloca, the fact that
3674 // the allocation is now on the stack is potentially visible, for example),
3675 // but we believe in a permissible manner.
3677
3678 // If we are removing an alloca with a dbg.declare, insert dbg.value calls
3679 // before each store.
3681 std::unique_ptr<DIBuilder> DIB;
3682 if (isa<AllocaInst>(MI)) {
3683 findDbgUsers(&MI, DVRs);
3684 DIB.reset(new DIBuilder(*MI.getModule(), /*AllowUnresolved=*/false));
3685 }
3686
3687 // Determine what getInitialValueOfAllocation would return without actually
3688 // allocating the result.
3689 bool KnowInitUndef = false;
3690 bool KnowInitZero = false;
3691 Constant *Init =
3693 if (Init) {
3694 if (isa<UndefValue>(Init))
3695 KnowInitUndef = true;
3696 else if (Init->isNullValue())
3697 KnowInitZero = true;
3698 }
3699 // The various sanitizers don't actually return undef memory, but rather
3700 // memory initialized with special forms of runtime poison
3701 auto &F = *MI.getFunction();
3702 if (F.hasFnAttribute(Attribute::SanitizeMemory) ||
3703 F.hasFnAttribute(Attribute::SanitizeAddress))
3704 KnowInitUndef = false;
3705
3706 auto Removable =
3707 isAllocSiteRemovable(&MI, Users, TLI, KnowInitZero | KnowInitUndef);
3708 if (Removable) {
3709 for (WeakTrackingVH &User : Users) {
3710 // Lowering all @llvm.objectsize and MTI calls first because they may use
3711 // a bitcast/GEP of the alloca we are removing.
3712 if (!User)
3713 continue;
3714
3716
3718 if (II->getIntrinsicID() == Intrinsic::objectsize) {
3719 SmallVector<Instruction *> InsertedInstructions;
3720 Value *Result = lowerObjectSizeCall(
3721 II, DL, &TLI, AA, /*MustSucceed=*/true, &InsertedInstructions);
3722 for (Instruction *Inserted : InsertedInstructions)
3723 Worklist.add(Inserted);
3724 replaceInstUsesWith(*I, Result);
3726 User = nullptr; // Skip examining in the next loop.
3727 continue;
3728 }
3729 if (auto *MTI = dyn_cast<MemTransferInst>(I)) {
3730 if (KnowInitZero && isRefSet(*Removable)) {
3732 Builder.SetInsertPoint(MTI);
3733 auto *M = Builder.CreateMemSet(
3734 MTI->getRawDest(),
3735 ConstantInt::get(Type::getInt8Ty(MI.getContext()), 0),
3736 MTI->getLength(), MTI->getDestAlign());
3737 M->copyMetadata(*MTI);
3738 }
3739 }
3740 }
3741 }
3742 for (WeakTrackingVH &User : Users) {
3743 if (!User)
3744 continue;
3745
3747
3748 if (ICmpInst *C = dyn_cast<ICmpInst>(I)) {
3750 ConstantInt::get(Type::getInt1Ty(C->getContext()),
3751 C->isFalseWhenEqual()));
3752 } else if (auto *SI = dyn_cast<StoreInst>(I)) {
3753 for (auto *DVR : DVRs)
3754 if (DVR->isAddressOfVariable())
3756 } else {
3757 // Casts, GEP, or anything else: we're about to delete this instruction,
3758 // so it can not have any valid uses.
3759 Constant *Replace;
3760 if (isa<LoadInst>(I)) {
3761 assert(KnowInitZero || KnowInitUndef);
3762 Replace = KnowInitUndef ? UndefValue::get(I->getType())
3763 : Constant::getNullValue(I->getType());
3764 } else
3765 Replace = PoisonValue::get(I->getType());
3766 replaceInstUsesWith(*I, Replace);
3767 }
3769 }
3770
3772 // Replace invoke with a NOP intrinsic to maintain the original CFG
3773 Module *M = II->getModule();
3774 Function *F = Intrinsic::getOrInsertDeclaration(M, Intrinsic::donothing);
3775 auto *NewII = InvokeInst::Create(
3776 F, II->getNormalDest(), II->getUnwindDest(), {}, "", II->getParent());
3777 NewII->setDebugLoc(II->getDebugLoc());
3778 }
3779
3780 // Remove debug intrinsics which describe the value contained within the
3781 // alloca. In addition to removing dbg.{declare,addr} which simply point to
3782 // the alloca, remove dbg.value(<alloca>, ..., DW_OP_deref)'s as well, e.g.:
3783 //
3784 // ```
3785 // define void @foo(i32 %0) {
3786 // %a = alloca i32 ; Deleted.
3787 // store i32 %0, i32* %a
3788 // dbg.value(i32 %0, "arg0") ; Not deleted.
3789 // dbg.value(i32* %a, "arg0", DW_OP_deref) ; Deleted.
3790 // call void @trivially_inlinable_no_op(i32* %a)
3791 // ret void
3792 // }
3793 // ```
3794 //
3795 // This may not be required if we stop describing the contents of allocas
3796 // using dbg.value(<alloca>, ..., DW_OP_deref), but we currently do this in
3797 // the LowerDbgDeclare utility.
3798 //
3799 // If there is a dead store to `%a` in @trivially_inlinable_no_op, the
3800 // "arg0" dbg.value may be stale after the call. However, failing to remove
3801 // the DW_OP_deref dbg.value causes large gaps in location coverage.
3802 //
3803 // FIXME: the Assignment Tracking project has now likely made this
3804 // redundant (and it's sometimes harmful).
3805 for (auto *DVR : DVRs)
3806 if (DVR->isAddressOfVariable() || DVR->getExpression()->startsWithDeref())
3807 DVR->eraseFromParent();
3808
3809 return eraseInstFromFunction(MI);
3810 }
3811 return nullptr;
3812}
3813
3814/// Move the call to free before a NULL test.
3815///
3816/// Check if this free is accessed after its argument has been test
3817/// against NULL (property 0).
3818/// If yes, it is legal to move this call in its predecessor block.
3819///
3820/// The move is performed only if the block containing the call to free
3821/// will be removed, i.e.:
3822/// 1. it has only one predecessor P, and P has two successors
3823/// 2. it contains the call, noops, and an unconditional branch
3824/// 3. its successor is the same as its predecessor's successor
3825///
3826/// The profitability is out-of concern here and this function should
3827/// be called only if the caller knows this transformation would be
3828/// profitable (e.g., for code size).
3830 const DataLayout &DL) {
3831 Value *Op = FI.getArgOperand(0);
3832 BasicBlock *FreeInstrBB = FI.getParent();
3833 BasicBlock *PredBB = FreeInstrBB->getSinglePredecessor();
3834
3835 // Validate part of constraint #1: Only one predecessor
3836 // FIXME: We can extend the number of predecessor, but in that case, we
3837 // would duplicate the call to free in each predecessor and it may
3838 // not be profitable even for code size.
3839 if (!PredBB)
3840 return nullptr;
3841
3842 // Validate constraint #2: Does this block contains only the call to
3843 // free, noops, and an unconditional branch?
3844 BasicBlock *SuccBB;
3845 Instruction *FreeInstrBBTerminator = FreeInstrBB->getTerminator();
3846 if (!match(FreeInstrBBTerminator, m_UnconditionalBr(SuccBB)))
3847 return nullptr;
3848
3849 // If there are only 2 instructions in the block, at this point,
3850 // this is the call to free and unconditional.
3851 // If there are more than 2 instructions, check that they are noops
3852 // i.e., they won't hurt the performance of the generated code.
3853 if (FreeInstrBB->size() != 2) {
3854 for (const Instruction &Inst : FreeInstrBB->instructionsWithoutDebug()) {
3855 if (&Inst == &FI || &Inst == FreeInstrBBTerminator)
3856 continue;
3857 auto *Cast = dyn_cast<CastInst>(&Inst);
3858 if (!Cast || !Cast->isNoopCast(DL))
3859 return nullptr;
3860 }
3861 }
3862 // Validate the rest of constraint #1 by matching on the pred branch.
3863 Instruction *TI = PredBB->getTerminator();
3864 BasicBlock *TrueBB, *FalseBB;
3865 CmpPredicate Pred;
3866 if (!match(TI, m_Br(m_ICmp(Pred,
3868 m_Specific(Op->stripPointerCasts())),
3869 m_Zero()),
3870 TrueBB, FalseBB)))
3871 return nullptr;
3872 if (Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
3873 return nullptr;
3874
3875 // Validate constraint #3: Ensure the null case just falls through.
3876 if (SuccBB != (Pred == ICmpInst::ICMP_EQ ? TrueBB : FalseBB))
3877 return nullptr;
3878 assert(FreeInstrBB == (Pred == ICmpInst::ICMP_EQ ? FalseBB : TrueBB) &&
3879 "Broken CFG: missing edge from predecessor to successor");
3880
3881 // At this point, we know that everything in FreeInstrBB can be moved
3882 // before TI.
3883 for (Instruction &Instr : llvm::make_early_inc_range(*FreeInstrBB)) {
3884 if (&Instr == FreeInstrBBTerminator)
3885 break;
3886 Instr.moveBeforePreserving(TI->getIterator());
3887 }
3888 assert(FreeInstrBB->size() == 1 &&
3889 "Only the branch instruction should remain");
3890
3891 // Now that we've moved the call to free before the NULL check, we have to
3892 // remove any attributes on its parameter that imply it's non-null, because
3893 // those attributes might have only been valid because of the NULL check, and
3894 // we can get miscompiles if we keep them. This is conservative if non-null is
3895 // also implied by something other than the NULL check, but it's guaranteed to
3896 // be correct, and the conservativeness won't matter in practice, since the
3897 // attributes are irrelevant for the call to free itself and the pointer
3898 // shouldn't be used after the call.
3899 AttributeList Attrs = FI.getAttributes();
3900 Attrs = Attrs.removeParamAttribute(FI.getContext(), 0, Attribute::NonNull);
3901 Attribute Dereferenceable = Attrs.getParamAttr(0, Attribute::Dereferenceable);
3902 if (Dereferenceable.isValid()) {
3903 uint64_t Bytes = Dereferenceable.getDereferenceableBytes();
3904 Attrs = Attrs.removeParamAttribute(FI.getContext(), 0,
3905 Attribute::Dereferenceable);
3906 Attrs = Attrs.addDereferenceableOrNullParamAttr(FI.getContext(), 0, Bytes);
3907 }
3908 FI.setAttributes(Attrs);
3909
3910 return &FI;
3911}
3912
3914 // free undef -> unreachable.
3915 if (isa<UndefValue>(Op)) {
3916 // Leave a marker since we can't modify the CFG here.
3918 return eraseInstFromFunction(FI);
3919 }
3920
3921 // If we have 'free null' delete the instruction. This can happen in stl code
3922 // when lots of inlining happens.
3924 return eraseInstFromFunction(FI);
3925
3926 // If we had free(realloc(...)) with no intervening uses, then eliminate the
3927 // realloc() entirely.
3929 if (CI && CI->hasOneUse())
3930 if (Value *ReallocatedOp = getReallocatedOperand(CI))
3931 return eraseInstFromFunction(*replaceInstUsesWith(*CI, ReallocatedOp));
3932
3933 // If we optimize for code size, try to move the call to free before the null
3934 // test so that simplify cfg can remove the empty block and dead code
3935 // elimination the branch. I.e., helps to turn something like:
3936 // if (foo) free(foo);
3937 // into
3938 // free(foo);
3939 //
3940 // Note that we can only do this for 'free' and not for any flavor of
3941 // 'operator delete'; there is no 'operator delete' symbol for which we are
3942 // permitted to invent a call, even if we're passing in a null pointer.
3943 if (MinimizeSize) {
3944 LibFunc Func;
3945 if (TLI.getLibFunc(FI, Func) && TLI.has(Func) && Func == LibFunc_free)
3947 return I;
3948 }
3949
3950 return nullptr;
3951}
3952
3954 Value *RetVal = RI.getReturnValue();
3955 if (!RetVal)
3956 return nullptr;
3957
3958 Function *F = RI.getFunction();
3959 Type *RetTy = RetVal->getType();
3960 if (RetTy->isPointerTy()) {
3961 bool HasDereferenceable =
3962 F->getAttributes().getRetDereferenceableBytes() > 0;
3963 if (F->hasRetAttribute(Attribute::NonNull) ||
3964 (HasDereferenceable &&
3966 if (Value *V = simplifyNonNullOperand(RetVal, HasDereferenceable))
3967 return replaceOperand(RI, 0, V);
3968 }
3969 }
3970
3971 if (!AttributeFuncs::isNoFPClassCompatibleType(RetTy))
3972 return nullptr;
3973
3974 FPClassTest ReturnClass = F->getAttributes().getRetNoFPClass();
3975 if (ReturnClass == fcNone)
3976 return nullptr;
3977
3978 KnownFPClass KnownClass;
3979 Value *Simplified =
3980 SimplifyDemandedUseFPClass(RetVal, ~ReturnClass, KnownClass, &RI);
3981 if (!Simplified)
3982 return nullptr;
3983
3984 return ReturnInst::Create(RI.getContext(), Simplified);
3985}
3986
3987// WARNING: keep in sync with SimplifyCFGOpt::simplifyUnreachable()!
3989 // Try to remove the previous instruction if it must lead to unreachable.
3990 // This includes instructions like stores and "llvm.assume" that may not get
3991 // removed by simple dead code elimination.
3992 bool Changed = false;
3993 while (Instruction *Prev = I.getPrevNode()) {
3994 // While we theoretically can erase EH, that would result in a block that
3995 // used to start with an EH no longer starting with EH, which is invalid.
3996 // To make it valid, we'd need to fixup predecessors to no longer refer to
3997 // this block, but that changes CFG, which is not allowed in InstCombine.
3998 if (Prev->isEHPad())
3999 break; // Can not drop any more instructions. We're done here.
4000
4002 break; // Can not drop any more instructions. We're done here.
4003 // Otherwise, this instruction can be freely erased,
4004 // even if it is not side-effect free.
4005
4006 // A value may still have uses before we process it here (for example, in
4007 // another unreachable block), so convert those to poison.
4008 replaceInstUsesWith(*Prev, PoisonValue::get(Prev->getType()));
4009 eraseInstFromFunction(*Prev);
4010 Changed = true;
4011 }
4012 return Changed;
4013}
4014
4019
4021 assert(BI.isUnconditional() && "Only for unconditional branches.");
4022
4023 // If this store is the second-to-last instruction in the basic block
4024 // (excluding debug info) and if the block ends with
4025 // an unconditional branch, try to move the store to the successor block.
4026
4027 auto GetLastSinkableStore = [](BasicBlock::iterator BBI) {
4028 BasicBlock::iterator FirstInstr = BBI->getParent()->begin();
4029 do {
4030 if (BBI != FirstInstr)
4031 --BBI;
4032 } while (BBI != FirstInstr && BBI->isDebugOrPseudoInst());
4033
4034 return dyn_cast<StoreInst>(BBI);
4035 };
4036
4037 if (StoreInst *SI = GetLastSinkableStore(BasicBlock::iterator(BI)))
4039 return &BI;
4040
4041 return nullptr;
4042}
4043
4046 if (!DeadEdges.insert({From, To}).second)
4047 return;
4048
4049 // Replace phi node operands in successor with poison.
4050 for (PHINode &PN : To->phis())
4051 for (Use &U : PN.incoming_values())
4052 if (PN.getIncomingBlock(U) == From && !isa<PoisonValue>(U)) {
4053 replaceUse(U, PoisonValue::get(PN.getType()));
4054 addToWorklist(&PN);
4055 MadeIRChange = true;
4056 }
4057
4058 Worklist.push_back(To);
4059}
4060
4061// Under the assumption that I is unreachable, remove it and following
4062// instructions. Changes are reported directly to MadeIRChange.
4065 BasicBlock *BB = I->getParent();
4066 for (Instruction &Inst : make_early_inc_range(
4067 make_range(std::next(BB->getTerminator()->getReverseIterator()),
4068 std::next(I->getReverseIterator())))) {
4069 if (!Inst.use_empty() && !Inst.getType()->isTokenTy()) {
4070 replaceInstUsesWith(Inst, PoisonValue::get(Inst.getType()));
4071 MadeIRChange = true;
4072 }
4073 if (Inst.isEHPad() || Inst.getType()->isTokenTy())
4074 continue;
4075 // RemoveDIs: erase debug-info on this instruction manually.
4076 Inst.dropDbgRecords();
4078 MadeIRChange = true;
4079 }
4080
4083 MadeIRChange = true;
4084 for (Value *V : Changed)
4086 }
4087
4088 // Handle potentially dead successors.
4089 for (BasicBlock *Succ : successors(BB))
4090 addDeadEdge(BB, Succ, Worklist);
4091}
4092
4095 while (!Worklist.empty()) {
4096 BasicBlock *BB = Worklist.pop_back_val();
4097 if (!all_of(predecessors(BB), [&](BasicBlock *Pred) {
4098 return DeadEdges.contains({Pred, BB}) || DT.dominates(BB, Pred);
4099 }))
4100 continue;
4101
4103 }
4104}
4105
4107 BasicBlock *LiveSucc) {
4109 for (BasicBlock *Succ : successors(BB)) {
4110 // The live successor isn't dead.
4111 if (Succ == LiveSucc)
4112 continue;
4113
4114 addDeadEdge(BB, Succ, Worklist);
4115 }
4116
4118}
4119
4121 if (BI.isUnconditional())
4123
4124 // Change br (not X), label True, label False to: br X, label False, True
4125 Value *Cond = BI.getCondition();
4126 Value *X;
4127 if (match(Cond, m_Not(m_Value(X))) && !isa<Constant>(X)) {
4128 // Swap Destinations and condition...
4129 BI.swapSuccessors();
4130 if (BPI)
4131 BPI->swapSuccEdgesProbabilities(BI.getParent());
4132 return replaceOperand(BI, 0, X);
4133 }
4134
4135 // Canonicalize logical-and-with-invert as logical-or-with-invert.
4136 // This is done by inverting the condition and swapping successors:
4137 // br (X && !Y), T, F --> br !(X && !Y), F, T --> br (!X || Y), F, T
4138 Value *Y;
4139 if (isa<SelectInst>(Cond) &&
4140 match(Cond,
4142 Value *NotX = Builder.CreateNot(X, "not." + X->getName());
4143 Value *Or = Builder.CreateLogicalOr(NotX, Y);
4144 BI.swapSuccessors();
4145 if (BPI)
4146 BPI->swapSuccEdgesProbabilities(BI.getParent());
4147 return replaceOperand(BI, 0, Or);
4148 }
4149
4150 // If the condition is irrelevant, remove the use so that other
4151 // transforms on the condition become more effective.
4152 if (!isa<ConstantInt>(Cond) && BI.getSuccessor(0) == BI.getSuccessor(1))
4153 return replaceOperand(BI, 0, ConstantInt::getFalse(Cond->getType()));
4154
4155 // Canonicalize, for example, fcmp_one -> fcmp_oeq.
4156 CmpPredicate Pred;
4157 if (match(Cond, m_OneUse(m_FCmp(Pred, m_Value(), m_Value()))) &&
4158 !isCanonicalPredicate(Pred)) {
4159 // Swap destinations and condition.
4160 auto *Cmp = cast<CmpInst>(Cond);
4161 Cmp->setPredicate(CmpInst::getInversePredicate(Pred));
4162 BI.swapSuccessors();
4163 if (BPI)
4164 BPI->swapSuccEdgesProbabilities(BI.getParent());
4165 Worklist.push(Cmp);
4166 return &BI;
4167 }
4168
4169 if (isa<UndefValue>(Cond)) {
4170 handlePotentiallyDeadSuccessors(BI.getParent(), /*LiveSucc*/ nullptr);
4171 return nullptr;
4172 }
4173 if (auto *CI = dyn_cast<ConstantInt>(Cond)) {
4175 BI.getSuccessor(!CI->getZExtValue()));
4176 return nullptr;
4177 }
4178
4179 // Replace all dominated uses of the condition with true/false
4180 // Ignore constant expressions to avoid iterating over uses on other
4181 // functions.
4182 if (!isa<Constant>(Cond) && BI.getSuccessor(0) != BI.getSuccessor(1)) {
4183 for (auto &U : make_early_inc_range(Cond->uses())) {
4184 BasicBlockEdge Edge0(BI.getParent(), BI.getSuccessor(0));
4185 if (DT.dominates(Edge0, U)) {
4186 replaceUse(U, ConstantInt::getTrue(Cond->getType()));
4187 addToWorklist(cast<Instruction>(U.getUser()));
4188 continue;
4189 }
4190 BasicBlockEdge Edge1(BI.getParent(), BI.getSuccessor(1));
4191 if (DT.dominates(Edge1, U)) {
4192 replaceUse(U, ConstantInt::getFalse(Cond->getType()));
4193 addToWorklist(cast<Instruction>(U.getUser()));
4194 }
4195 }
4196 }
4197
4198 DC.registerBranch(&BI);
4199 return nullptr;
4200}
4201
4202// Replaces (switch (select cond, X, C)/(select cond, C, X)) with (switch X) if
4203// we can prove that both (switch C) and (switch X) go to the default when cond
4204// is false/true.
4207 bool IsTrueArm) {
4208 unsigned CstOpIdx = IsTrueArm ? 1 : 2;
4209 auto *C = dyn_cast<ConstantInt>(Select->getOperand(CstOpIdx));
4210 if (!C)
4211 return nullptr;
4212
4213 BasicBlock *CstBB = SI.findCaseValue(C)->getCaseSuccessor();
4214 if (CstBB != SI.getDefaultDest())
4215 return nullptr;
4216 Value *X = Select->getOperand(3 - CstOpIdx);
4217 CmpPredicate Pred;
4218 const APInt *RHSC;
4219 if (!match(Select->getCondition(),
4220 m_ICmp(Pred, m_Specific(X), m_APInt(RHSC))))
4221 return nullptr;
4222 if (IsTrueArm)
4223 Pred = ICmpInst::getInversePredicate(Pred);
4224
4225 // See whether we can replace the select with X
4227 for (auto Case : SI.cases())
4228 if (!CR.contains(Case.getCaseValue()->getValue()))
4229 return nullptr;
4230
4231 return X;
4232}
4233
4235 Value *Cond = SI.getCondition();
4236 Value *Op0;
4237 ConstantInt *AddRHS;
4238 if (match(Cond, m_Add(m_Value(Op0), m_ConstantInt(AddRHS)))) {
4239 // Change 'switch (X+4) case 1:' into 'switch (X) case -3'.
4240 for (auto Case : SI.cases()) {
4241 Constant *NewCase = ConstantExpr::getSub(Case.getCaseValue(), AddRHS);
4242 assert(isa<ConstantInt>(NewCase) &&
4243 "Result of expression should be constant");
4244 Case.setValue(cast<ConstantInt>(NewCase));
4245 }
4246 return replaceOperand(SI, 0, Op0);
4247 }
4248
4249 ConstantInt *SubLHS;
4250 if (match(Cond, m_Sub(m_ConstantInt(SubLHS), m_Value(Op0)))) {
4251 // Change 'switch (1-X) case 1:' into 'switch (X) case 0'.
4252 for (auto Case : SI.cases()) {
4253 Constant *NewCase = ConstantExpr::getSub(SubLHS, Case.getCaseValue());
4254 assert(isa<ConstantInt>(NewCase) &&
4255 "Result of expression should be constant");
4256 Case.setValue(cast<ConstantInt>(NewCase));
4257 }
4258 return replaceOperand(SI, 0, Op0);
4259 }
4260
4261 uint64_t ShiftAmt;
4262 if (match(Cond, m_Shl(m_Value(Op0), m_ConstantInt(ShiftAmt))) &&
4263 ShiftAmt < Op0->getType()->getScalarSizeInBits() &&
4264 all_of(SI.cases(), [&](const auto &Case) {
4265 return Case.getCaseValue()->getValue().countr_zero() >= ShiftAmt;
4266 })) {
4267 // Change 'switch (X << 2) case 4:' into 'switch (X) case 1:'.
4269 if (Shl->hasNoUnsignedWrap() || Shl->hasNoSignedWrap() ||
4270 Shl->hasOneUse()) {
4271 Value *NewCond = Op0;
4272 if (!Shl->hasNoUnsignedWrap() && !Shl->hasNoSignedWrap()) {
4273 // If the shift may wrap, we need to mask off the shifted bits.
4274 unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
4275 NewCond = Builder.CreateAnd(
4276 Op0, APInt::getLowBitsSet(BitWidth, BitWidth - ShiftAmt));
4277 }
4278 for (auto Case : SI.cases()) {
4279 const APInt &CaseVal = Case.getCaseValue()->getValue();
4280 APInt ShiftedCase = Shl->hasNoSignedWrap() ? CaseVal.ashr(ShiftAmt)
4281 : CaseVal.lshr(ShiftAmt);
4282 Case.setValue(ConstantInt::get(SI.getContext(), ShiftedCase));
4283 }
4284 return replaceOperand(SI, 0, NewCond);
4285 }
4286 }
4287
4288 // Fold switch(zext/sext(X)) into switch(X) if possible.
4289 if (match(Cond, m_ZExtOrSExt(m_Value(Op0)))) {
4290 bool IsZExt = isa<ZExtInst>(Cond);
4291 Type *SrcTy = Op0->getType();
4292 unsigned NewWidth = SrcTy->getScalarSizeInBits();
4293
4294 if (all_of(SI.cases(), [&](const auto &Case) {
4295 const APInt &CaseVal = Case.getCaseValue()->getValue();
4296 return IsZExt ? CaseVal.isIntN(NewWidth)
4297 : CaseVal.isSignedIntN(NewWidth);
4298 })) {
4299 for (auto &Case : SI.cases()) {
4300 APInt TruncatedCase = Case.getCaseValue()->getValue().trunc(NewWidth);
4301 Case.setValue(ConstantInt::get(SI.getContext(), TruncatedCase));
4302 }
4303 return replaceOperand(SI, 0, Op0);
4304 }
4305 }
4306
4307 // Fold switch(select cond, X, Y) into switch(X/Y) if possible
4308 if (auto *Select = dyn_cast<SelectInst>(Cond)) {
4309 if (Value *V =
4310 simplifySwitchOnSelectUsingRanges(SI, Select, /*IsTrueArm=*/true))
4311 return replaceOperand(SI, 0, V);
4312 if (Value *V =
4313 simplifySwitchOnSelectUsingRanges(SI, Select, /*IsTrueArm=*/false))
4314 return replaceOperand(SI, 0, V);
4315 }
4316
4317 KnownBits Known = computeKnownBits(Cond, &SI);
4318 unsigned LeadingKnownZeros = Known.countMinLeadingZeros();
4319 unsigned LeadingKnownOnes = Known.countMinLeadingOnes();
4320
4321 // Compute the number of leading bits we can ignore.
4322 // TODO: A better way to determine this would use ComputeNumSignBits().
4323 for (const auto &C : SI.cases()) {
4324 LeadingKnownZeros =
4325 std::min(LeadingKnownZeros, C.getCaseValue()->getValue().countl_zero());
4326 LeadingKnownOnes =
4327 std::min(LeadingKnownOnes, C.getCaseValue()->getValue().countl_one());
4328 }
4329
4330 unsigned NewWidth = Known.getBitWidth() - std::max(LeadingKnownZeros, LeadingKnownOnes);
4331
4332 // Shrink the condition operand if the new type is smaller than the old type.
4333 // But do not shrink to a non-standard type, because backend can't generate
4334 // good code for that yet.
4335 // TODO: We can make it aggressive again after fixing PR39569.
4336 if (NewWidth > 0 && NewWidth < Known.getBitWidth() &&
4337 shouldChangeType(Known.getBitWidth(), NewWidth)) {
4338 IntegerType *Ty = IntegerType::get(SI.getContext(), NewWidth);
4339 Builder.SetInsertPoint(&SI);
4340 Value *NewCond = Builder.CreateTrunc(Cond, Ty, "trunc");
4341
4342 for (auto Case : SI.cases()) {
4343 APInt TruncatedCase = Case.getCaseValue()->getValue().trunc(NewWidth);
4344 Case.setValue(ConstantInt::get(SI.getContext(), TruncatedCase));
4345 }
4346 return replaceOperand(SI, 0, NewCond);
4347 }
4348
4349 if (isa<UndefValue>(Cond)) {
4350 handlePotentiallyDeadSuccessors(SI.getParent(), /*LiveSucc*/ nullptr);
4351 return nullptr;
4352 }
4353 if (auto *CI = dyn_cast<ConstantInt>(Cond)) {
4355 SI.findCaseValue(CI)->getCaseSuccessor());
4356 return nullptr;
4357 }
4358
4359 return nullptr;
4360}
4361
4363InstCombinerImpl::foldExtractOfOverflowIntrinsic(ExtractValueInst &EV) {
4365 if (!WO)
4366 return nullptr;
4367
4368 Intrinsic::ID OvID = WO->getIntrinsicID();
4369 const APInt *C = nullptr;
4370 if (match(WO->getRHS(), m_APIntAllowPoison(C))) {
4371 if (*EV.idx_begin() == 0 && (OvID == Intrinsic::smul_with_overflow ||
4372 OvID == Intrinsic::umul_with_overflow)) {
4373 // extractvalue (any_mul_with_overflow X, -1), 0 --> -X
4374 if (C->isAllOnes())
4375 return BinaryOperator::CreateNeg(WO->getLHS());
4376 // extractvalue (any_mul_with_overflow X, 2^n), 0 --> X << n
4377 if (C->isPowerOf2()) {
4378 return BinaryOperator::CreateShl(
4379 WO->getLHS(),
4380 ConstantInt::get(WO->getLHS()->getType(), C->logBase2()));
4381 }
4382 }
4383 }
4384
4385 // We're extracting from an overflow intrinsic. See if we're the only user.
4386 // That allows us to simplify multiple result intrinsics to simpler things
4387 // that just get one value.
4388 if (!WO->hasOneUse())
4389 return nullptr;
4390
4391 // Check if we're grabbing only the result of a 'with overflow' intrinsic
4392 // and replace it with a traditional binary instruction.
4393 if (*EV.idx_begin() == 0) {
4394 Instruction::BinaryOps BinOp = WO->getBinaryOp();
4395 Value *LHS = WO->getLHS(), *RHS = WO->getRHS();
4396 // Replace the old instruction's uses with poison.
4397 replaceInstUsesWith(*WO, PoisonValue::get(WO->getType()));
4399 return BinaryOperator::Create(BinOp, LHS, RHS);
4400 }
4401
4402 assert(*EV.idx_begin() == 1 && "Unexpected extract index for overflow inst");
4403
4404 // (usub LHS, RHS) overflows when LHS is unsigned-less-than RHS.
4405 if (OvID == Intrinsic::usub_with_overflow)
4406 return new ICmpInst(ICmpInst::ICMP_ULT, WO->getLHS(), WO->getRHS());
4407
4408 // smul with i1 types overflows when both sides are set: -1 * -1 == +1, but
4409 // +1 is not possible because we assume signed values.
4410 if (OvID == Intrinsic::smul_with_overflow &&
4411 WO->getLHS()->getType()->isIntOrIntVectorTy(1))
4412 return BinaryOperator::CreateAnd(WO->getLHS(), WO->getRHS());
4413
4414 // extractvalue (umul_with_overflow X, X), 1 -> X u> 2^(N/2)-1
4415 if (OvID == Intrinsic::umul_with_overflow && WO->getLHS() == WO->getRHS()) {
4416 unsigned BitWidth = WO->getLHS()->getType()->getScalarSizeInBits();
4417 // Only handle even bitwidths for performance reasons.
4418 if (BitWidth % 2 == 0)
4419 return new ICmpInst(
4420 ICmpInst::ICMP_UGT, WO->getLHS(),
4421 ConstantInt::get(WO->getLHS()->getType(),
4423 }
4424
4425 // If only the overflow result is used, and the right hand side is a
4426 // constant (or constant splat), we can remove the intrinsic by directly
4427 // checking for overflow.
4428 if (C) {
4429 // Compute the no-wrap range for LHS given RHS=C, then construct an
4430 // equivalent icmp, potentially using an offset.
4431 ConstantRange NWR = ConstantRange::makeExactNoWrapRegion(
4432 WO->getBinaryOp(), *C, WO->getNoWrapKind());
4433
4434 CmpInst::Predicate Pred;
4435 APInt NewRHSC, Offset;
4436 NWR.getEquivalentICmp(Pred, NewRHSC, Offset);
4437 auto *OpTy = WO->getRHS()->getType();
4438 auto *NewLHS = WO->getLHS();
4439 if (Offset != 0)
4440 NewLHS = Builder.CreateAdd(NewLHS, ConstantInt::get(OpTy, Offset));
4441 return new ICmpInst(ICmpInst::getInversePredicate(Pred), NewLHS,
4442 ConstantInt::get(OpTy, NewRHSC));
4443 }
4444
4445 return nullptr;
4446}
4447
4450 InstCombiner::BuilderTy &Builder) {
4451 // Helper to fold frexp of select to select of frexp.
4452
4453 if (!SelectInst->hasOneUse() || !FrexpCall->hasOneUse())
4454 return nullptr;
4456 Value *TrueVal = SelectInst->getTrueValue();
4457 Value *FalseVal = SelectInst->getFalseValue();
4458
4459 const APFloat *ConstVal = nullptr;
4460 Value *VarOp = nullptr;
4461 bool ConstIsTrue = false;
4462
4463 if (match(TrueVal, m_APFloat(ConstVal))) {
4464 VarOp = FalseVal;
4465 ConstIsTrue = true;
4466 } else if (match(FalseVal, m_APFloat(ConstVal))) {
4467 VarOp = TrueVal;
4468 ConstIsTrue = false;
4469 } else {
4470 return nullptr;
4471 }
4472
4473 Builder.SetInsertPoint(&EV);
4474
4475 CallInst *NewFrexp =
4476 Builder.CreateCall(FrexpCall->getCalledFunction(), {VarOp}, "frexp");
4477 NewFrexp->copyIRFlags(FrexpCall);
4478
4479 Value *NewEV = Builder.CreateExtractValue(NewFrexp, 0, "mantissa");
4480
4481 int Exp;
4482 APFloat Mantissa = frexp(*ConstVal, Exp, APFloat::rmNearestTiesToEven);
4483
4484 Constant *ConstantMantissa = ConstantFP::get(TrueVal->getType(), Mantissa);
4485
4486 Value *NewSel = Builder.CreateSelectFMF(
4487 Cond, ConstIsTrue ? ConstantMantissa : NewEV,
4488 ConstIsTrue ? NewEV : ConstantMantissa, SelectInst, "select.frexp");
4489 return NewSel;
4490}
4492 Value *Agg = EV.getAggregateOperand();
4493
4494 if (!EV.hasIndices())
4495 return replaceInstUsesWith(EV, Agg);
4496
4497 if (Value *V = simplifyExtractValueInst(Agg, EV.getIndices(),
4498 SQ.getWithInstruction(&EV)))
4499 return replaceInstUsesWith(EV, V);
4500
4501 Value *Cond, *TrueVal, *FalseVal;
4503 m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal)))))) {
4504 auto *SelInst =
4505 cast<SelectInst>(cast<IntrinsicInst>(Agg)->getArgOperand(0));
4506 if (Value *Result =
4507 foldFrexpOfSelect(EV, cast<IntrinsicInst>(Agg), SelInst, Builder))
4508 return replaceInstUsesWith(EV, Result);
4509 }
4511 // We're extracting from an insertvalue instruction, compare the indices
4512 const unsigned *exti, *exte, *insi, *inse;
4513 for (exti = EV.idx_begin(), insi = IV->idx_begin(),
4514 exte = EV.idx_end(), inse = IV->idx_end();
4515 exti != exte && insi != inse;
4516 ++exti, ++insi) {
4517 if (*insi != *exti)
4518 // The insert and extract both reference distinctly different elements.
4519 // This means the extract is not influenced by the insert, and we can
4520 // replace the aggregate operand of the extract with the aggregate
4521 // operand of the insert. i.e., replace
4522 // %I = insertvalue { i32, { i32 } } %A, { i32 } { i32 42 }, 1
4523 // %E = extractvalue { i32, { i32 } } %I, 0
4524 // with
4525 // %E = extractvalue { i32, { i32 } } %A, 0
4526 return ExtractValueInst::Create(IV->getAggregateOperand(),
4527 EV.getIndices());
4528 }
4529 if (exti == exte && insi == inse)
4530 // Both iterators are at the end: Index lists are identical. Replace
4531 // %B = insertvalue { i32, { i32 } } %A, i32 42, 1, 0
4532 // %C = extractvalue { i32, { i32 } } %B, 1, 0
4533 // with "i32 42"
4534 return replaceInstUsesWith(EV, IV->getInsertedValueOperand());
4535 if (exti == exte) {
4536 // The extract list is a prefix of the insert list. i.e. replace
4537 // %I = insertvalue { i32, { i32 } } %A, i32 42, 1, 0
4538 // %E = extractvalue { i32, { i32 } } %I, 1
4539 // with
4540 // %X = extractvalue { i32, { i32 } } %A, 1
4541 // %E = insertvalue { i32 } %X, i32 42, 0
4542 // by switching the order of the insert and extract (though the
4543 // insertvalue should be left in, since it may have other uses).
4544 Value *NewEV = Builder.CreateExtractValue(IV->getAggregateOperand(),
4545 EV.getIndices());
4546 return InsertValueInst::Create(NewEV, IV->getInsertedValueOperand(),
4547 ArrayRef(insi, inse));
4548 }
4549 if (insi == inse)
4550 // The insert list is a prefix of the extract list
4551 // We can simply remove the common indices from the extract and make it
4552 // operate on the inserted value instead of the insertvalue result.
4553 // i.e., replace
4554 // %I = insertvalue { i32, { i32 } } %A, { i32 } { i32 42 }, 1
4555 // %E = extractvalue { i32, { i32 } } %I, 1, 0
4556 // with
4557 // %E extractvalue { i32 } { i32 42 }, 0
4558 return ExtractValueInst::Create(IV->getInsertedValueOperand(),
4559 ArrayRef(exti, exte));
4560 }
4561
4562 if (Instruction *R = foldExtractOfOverflowIntrinsic(EV))
4563 return R;
4564
4565 if (LoadInst *L = dyn_cast<LoadInst>(Agg)) {
4566 // Bail out if the aggregate contains scalable vector type
4567 if (auto *STy = dyn_cast<StructType>(Agg->getType());
4568 STy && STy->isScalableTy())
4569 return nullptr;
4570
4571 // If the (non-volatile) load only has one use, we can rewrite this to a
4572 // load from a GEP. This reduces the size of the load. If a load is used
4573 // only by extractvalue instructions then this either must have been
4574 // optimized before, or it is a struct with padding, in which case we
4575 // don't want to do the transformation as it loses padding knowledge.
4576 if (L->isSimple() && L->hasOneUse()) {
4577 // extractvalue has integer indices, getelementptr has Value*s. Convert.
4578 SmallVector<Value*, 4> Indices;
4579 // Prefix an i32 0 since we need the first element.
4580 Indices.push_back(Builder.getInt32(0));
4581 for (unsigned Idx : EV.indices())
4582 Indices.push_back(Builder.getInt32(Idx));
4583
4584 // We need to insert these at the location of the old load, not at that of
4585 // the extractvalue.
4586 Builder.SetInsertPoint(L);
4587 Value *GEP = Builder.CreateInBoundsGEP(L->getType(),
4588 L->getPointerOperand(), Indices);
4589 Instruction *NL = Builder.CreateLoad(EV.getType(), GEP);
4590 // Whatever aliasing information we had for the orignal load must also
4591 // hold for the smaller load, so propagate the annotations.
4592 NL->setAAMetadata(L->getAAMetadata());
4593 // Returning the load directly will cause the main loop to insert it in
4594 // the wrong spot, so use replaceInstUsesWith().
4595 return replaceInstUsesWith(EV, NL);
4596 }
4597 }
4598
4599 if (auto *PN = dyn_cast<PHINode>(Agg))
4600 if (Instruction *Res = foldOpIntoPhi(EV, PN))
4601 return Res;
4602
4603 // Canonicalize extract (select Cond, TV, FV)
4604 // -> select cond, (extract TV), (extract FV)
4605 if (auto *SI = dyn_cast<SelectInst>(Agg))
4606 if (Instruction *R = FoldOpIntoSelect(EV, SI, /*FoldWithMultiUse=*/true))
4607 return R;
4608
4609 // We could simplify extracts from other values. Note that nested extracts may
4610 // already be simplified implicitly by the above: extract (extract (insert) )
4611 // will be translated into extract ( insert ( extract ) ) first and then just
4612 // the value inserted, if appropriate. Similarly for extracts from single-use
4613 // loads: extract (extract (load)) will be translated to extract (load (gep))
4614 // and if again single-use then via load (gep (gep)) to load (gep).
4615 // However, double extracts from e.g. function arguments or return values
4616 // aren't handled yet.
4617 return nullptr;
4618}
4619
4620/// Return 'true' if the given typeinfo will match anything.
4621static bool isCatchAll(EHPersonality Personality, Constant *TypeInfo) {
4622 switch (Personality) {
4626 // The GCC C EH and Rust personality only exists to support cleanups, so
4627 // it's not clear what the semantics of catch clauses are.
4628 return false;
4630 return false;
4632 // While __gnat_all_others_value will match any Ada exception, it doesn't
4633 // match foreign exceptions (or didn't, before gcc-4.7).
4634 return false;
4645 return TypeInfo->isNullValue();
4646 }
4647 llvm_unreachable("invalid enum");
4648}
4649
4650static bool shorter_filter(const Value *LHS, const Value *RHS) {
4651 return
4652 cast<ArrayType>(LHS->getType())->getNumElements()
4653 <
4654 cast<ArrayType>(RHS->getType())->getNumElements();
4655}
4656
4658 // The logic here should be correct for any real-world personality function.
4659 // However if that turns out not to be true, the offending logic can always
4660 // be conditioned on the personality function, like the catch-all logic is.
4661 EHPersonality Personality =
4662 classifyEHPersonality(LI.getParent()->getParent()->getPersonalityFn());
4663
4664 // Simplify the list of clauses, eg by removing repeated catch clauses
4665 // (these are often created by inlining).
4666 bool MakeNewInstruction = false; // If true, recreate using the following:
4667 SmallVector<Constant *, 16> NewClauses; // - Clauses for the new instruction;
4668 bool CleanupFlag = LI.isCleanup(); // - The new instruction is a cleanup.
4669
4670 SmallPtrSet<Value *, 16> AlreadyCaught; // Typeinfos known caught already.
4671 for (unsigned i = 0, e = LI.getNumClauses(); i != e; ++i) {
4672 bool isLastClause = i + 1 == e;
4673 if (LI.isCatch(i)) {
4674 // A catch clause.
4675 Constant *CatchClause = LI.getClause(i);
4676 Constant *TypeInfo = CatchClause->stripPointerCasts();
4677
4678 // If we already saw this clause, there is no point in having a second
4679 // copy of it.
4680 if (AlreadyCaught.insert(TypeInfo).second) {
4681 // This catch clause was not already seen.
4682 NewClauses.push_back(CatchClause);
4683 } else {
4684 // Repeated catch clause - drop the redundant copy.
4685 MakeNewInstruction = true;
4686 }
4687
4688 // If this is a catch-all then there is no point in keeping any following
4689 // clauses or marking the landingpad as having a cleanup.
4690 if (isCatchAll(Personality, TypeInfo)) {
4691 if (!isLastClause)
4692 MakeNewInstruction = true;
4693 CleanupFlag = false;
4694 break;
4695 }
4696 } else {
4697 // A filter clause. If any of the filter elements were already caught
4698 // then they can be dropped from the filter. It is tempting to try to
4699 // exploit the filter further by saying that any typeinfo that does not
4700 // occur in the filter can't be caught later (and thus can be dropped).
4701 // However this would be wrong, since typeinfos can match without being
4702 // equal (for example if one represents a C++ class, and the other some
4703 // class derived from it).
4704 assert(LI.isFilter(i) && "Unsupported landingpad clause!");
4705 Constant *FilterClause = LI.getClause(i);
4706 ArrayType *FilterType = cast<ArrayType>(FilterClause->getType());
4707 unsigned NumTypeInfos = FilterType->getNumElements();
4708
4709 // An empty filter catches everything, so there is no point in keeping any
4710 // following clauses or marking the landingpad as having a cleanup. By
4711 // dealing with this case here the following code is made a bit simpler.
4712 if (!NumTypeInfos) {
4713 NewClauses.push_back(FilterClause);
4714 if (!isLastClause)
4715 MakeNewInstruction = true;
4716 CleanupFlag = false;
4717 break;
4718 }
4719
4720 bool MakeNewFilter = false; // If true, make a new filter.
4721 SmallVector<Constant *, 16> NewFilterElts; // New elements.
4722 if (isa<ConstantAggregateZero>(FilterClause)) {
4723 // Not an empty filter - it contains at least one null typeinfo.
4724 assert(NumTypeInfos > 0 && "Should have handled empty filter already!");
4725 Constant *TypeInfo =
4727 // If this typeinfo is a catch-all then the filter can never match.
4728 if (isCatchAll(Personality, TypeInfo)) {
4729 // Throw the filter away.
4730 MakeNewInstruction = true;
4731 continue;
4732 }
4733
4734 // There is no point in having multiple copies of this typeinfo, so
4735 // discard all but the first copy if there is more than one.
4736 NewFilterElts.push_back(TypeInfo);
4737 if (NumTypeInfos > 1)
4738 MakeNewFilter = true;
4739 } else {
4740 ConstantArray *Filter = cast<ConstantArray>(FilterClause);
4741 SmallPtrSet<Value *, 16> SeenInFilter; // For uniquing the elements.
4742 NewFilterElts.reserve(NumTypeInfos);
4743
4744 // Remove any filter elements that were already caught or that already
4745 // occurred in the filter. While there, see if any of the elements are
4746 // catch-alls. If so, the filter can be discarded.
4747 bool SawCatchAll = false;
4748 for (unsigned j = 0; j != NumTypeInfos; ++j) {
4749 Constant *Elt = Filter->getOperand(j);
4750 Constant *TypeInfo = Elt->stripPointerCasts();
4751 if (isCatchAll(Personality, TypeInfo)) {
4752 // This element is a catch-all. Bail out, noting this fact.
4753 SawCatchAll = true;
4754 break;
4755 }
4756
4757 // Even if we've seen a type in a catch clause, we don't want to
4758 // remove it from the filter. An unexpected type handler may be
4759 // set up for a call site which throws an exception of the same
4760 // type caught. In order for the exception thrown by the unexpected
4761 // handler to propagate correctly, the filter must be correctly
4762 // described for the call site.
4763 //
4764 // Example:
4765 //
4766 // void unexpected() { throw 1;}
4767 // void foo() throw (int) {
4768 // std::set_unexpected(unexpected);
4769 // try {
4770 // throw 2.0;
4771 // } catch (int i) {}
4772 // }
4773
4774 // There is no point in having multiple copies of the same typeinfo in
4775 // a filter, so only add it if we didn't already.
4776 if (SeenInFilter.insert(TypeInfo).second)
4777 NewFilterElts.push_back(cast<Constant>(Elt));
4778 }
4779 // A filter containing a catch-all cannot match anything by definition.
4780 if (SawCatchAll) {
4781 // Throw the filter away.
4782 MakeNewInstruction = true;
4783 continue;
4784 }
4785
4786 // If we dropped something from the filter, make a new one.
4787 if (NewFilterElts.size() < NumTypeInfos)
4788 MakeNewFilter = true;
4789 }
4790 if (MakeNewFilter) {
4791 FilterType = ArrayType::get(FilterType->getElementType(),
4792 NewFilterElts.size());
4793 FilterClause = ConstantArray::get(FilterType, NewFilterElts);
4794 MakeNewInstruction = true;
4795 }
4796
4797 NewClauses.push_back(FilterClause);
4798
4799 // If the new filter is empty then it will catch everything so there is
4800 // no point in keeping any following clauses or marking the landingpad
4801 // as having a cleanup. The case of the original filter being empty was
4802 // already handled above.
4803 if (MakeNewFilter && !NewFilterElts.size()) {
4804 assert(MakeNewInstruction && "New filter but not a new instruction!");
4805 CleanupFlag = false;
4806 break;
4807 }
4808 }
4809 }
4810
4811 // If several filters occur in a row then reorder them so that the shortest
4812 // filters come first (those with the smallest number of elements). This is
4813 // advantageous because shorter filters are more likely to match, speeding up
4814 // unwinding, but mostly because it increases the effectiveness of the other
4815 // filter optimizations below.
4816 for (unsigned i = 0, e = NewClauses.size(); i + 1 < e; ) {
4817 unsigned j;
4818 // Find the maximal 'j' s.t. the range [i, j) consists entirely of filters.
4819 for (j = i; j != e; ++j)
4820 if (!isa<ArrayType>(NewClauses[j]->getType()))
4821 break;
4822
4823 // Check whether the filters are already sorted by length. We need to know
4824 // if sorting them is actually going to do anything so that we only make a
4825 // new landingpad instruction if it does.
4826 for (unsigned k = i; k + 1 < j; ++k)
4827 if (shorter_filter(NewClauses[k+1], NewClauses[k])) {
4828 // Not sorted, so sort the filters now. Doing an unstable sort would be
4829 // correct too but reordering filters pointlessly might confuse users.
4830 std::stable_sort(NewClauses.begin() + i, NewClauses.begin() + j,
4832 MakeNewInstruction = true;
4833 break;
4834 }
4835
4836 // Look for the next batch of filters.
4837 i = j + 1;
4838 }
4839
4840 // If typeinfos matched if and only if equal, then the elements of a filter L
4841 // that occurs later than a filter F could be replaced by the intersection of
4842 // the elements of F and L. In reality two typeinfos can match without being
4843 // equal (for example if one represents a C++ class, and the other some class
4844 // derived from it) so it would be wrong to perform this transform in general.
4845 // However the transform is correct and useful if F is a subset of L. In that
4846 // case L can be replaced by F, and thus removed altogether since repeating a
4847 // filter is pointless. So here we look at all pairs of filters F and L where
4848 // L follows F in the list of clauses, and remove L if every element of F is
4849 // an element of L. This can occur when inlining C++ functions with exception
4850 // specifications.
4851 for (unsigned i = 0; i + 1 < NewClauses.size(); ++i) {
4852 // Examine each filter in turn.
4853 Value *Filter = NewClauses[i];
4854 ArrayType *FTy = dyn_cast<ArrayType>(Filter->getType());
4855 if (!FTy)
4856 // Not a filter - skip it.
4857 continue;
4858 unsigned FElts = FTy->getNumElements();
4859 // Examine each filter following this one. Doing this backwards means that
4860 // we don't have to worry about filters disappearing under us when removed.
4861 for (unsigned j = NewClauses.size() - 1; j != i; --j) {
4862 Value *LFilter = NewClauses[j];
4863 ArrayType *LTy = dyn_cast<ArrayType>(LFilter->getType());
4864 if (!LTy)
4865 // Not a filter - skip it.
4866 continue;
4867 // If Filter is a subset of LFilter, i.e. every element of Filter is also
4868 // an element of LFilter, then discard LFilter.
4869 SmallVectorImpl<Constant *>::iterator J = NewClauses.begin() + j;
4870 // If Filter is empty then it is a subset of LFilter.
4871 if (!FElts) {
4872 // Discard LFilter.
4873 NewClauses.erase(J);
4874 MakeNewInstruction = true;
4875 // Move on to the next filter.
4876 continue;
4877 }
4878 unsigned LElts = LTy->getNumElements();
4879 // If Filter is longer than LFilter then it cannot be a subset of it.
4880 if (FElts > LElts)
4881 // Move on to the next filter.
4882 continue;
4883 // At this point we know that LFilter has at least one element.
4884 if (isa<ConstantAggregateZero>(LFilter)) { // LFilter only contains zeros.
4885 // Filter is a subset of LFilter iff Filter contains only zeros (as we
4886 // already know that Filter is not longer than LFilter).
4888 assert(FElts <= LElts && "Should have handled this case earlier!");
4889 // Discard LFilter.
4890 NewClauses.erase(J);
4891 MakeNewInstruction = true;
4892 }
4893 // Move on to the next filter.
4894 continue;
4895 }
4896 ConstantArray *LArray = cast<ConstantArray>(LFilter);
4897 if (isa<ConstantAggregateZero>(Filter)) { // Filter only contains zeros.
4898 // Since Filter is non-empty and contains only zeros, it is a subset of
4899 // LFilter iff LFilter contains a zero.
4900 assert(FElts > 0 && "Should have eliminated the empty filter earlier!");
4901 for (unsigned l = 0; l != LElts; ++l)
4902 if (LArray->getOperand(l)->isNullValue()) {
4903 // LFilter contains a zero - discard it.
4904 NewClauses.erase(J);
4905 MakeNewInstruction = true;
4906 break;
4907 }
4908 // Move on to the next filter.
4909 continue;
4910 }
4911 // At this point we know that both filters are ConstantArrays. Loop over
4912 // operands to see whether every element of Filter is also an element of
4913 // LFilter. Since filters tend to be short this is probably faster than
4914 // using a method that scales nicely.
4916 bool AllFound = true;
4917 for (unsigned f = 0; f != FElts; ++f) {
4918 Value *FTypeInfo = FArray->getOperand(f)->stripPointerCasts();
4919 AllFound = false;
4920 for (unsigned l = 0; l != LElts; ++l) {
4921 Value *LTypeInfo = LArray->getOperand(l)->stripPointerCasts();
4922 if (LTypeInfo == FTypeInfo) {
4923 AllFound = true;
4924 break;
4925 }
4926 }
4927 if (!AllFound)
4928 break;
4929 }
4930 if (AllFound) {
4931 // Discard LFilter.
4932 NewClauses.erase(J);
4933 MakeNewInstruction = true;
4934 }
4935 // Move on to the next filter.
4936 }
4937 }
4938
4939 // If we changed any of the clauses, replace the old landingpad instruction
4940 // with a new one.
4941 if (MakeNewInstruction) {
4943 NewClauses.size());
4944 for (Constant *C : NewClauses)
4945 NLI->addClause(C);
4946 // A landing pad with no clauses must have the cleanup flag set. It is
4947 // theoretically possible, though highly unlikely, that we eliminated all
4948 // clauses. If so, force the cleanup flag to true.
4949 if (NewClauses.empty())
4950 CleanupFlag = true;
4951 NLI->setCleanup(CleanupFlag);
4952 return NLI;
4953 }
4954
4955 // Even if none of the clauses changed, we may nonetheless have understood
4956 // that the cleanup flag is pointless. Clear it if so.
4957 if (LI.isCleanup() != CleanupFlag) {
4958 assert(!CleanupFlag && "Adding a cleanup, not removing one?!");
4959 LI.setCleanup(CleanupFlag);
4960 return &LI;
4961 }
4962
4963 return nullptr;
4964}
4965
4966Value *
4968 // Try to push freeze through instructions that propagate but don't produce
4969 // poison as far as possible. If an operand of freeze does not produce poison
4970 // then push the freeze through to the operands that are not guaranteed
4971 // non-poison. The actual transform is as follows.
4972 // Op1 = ... ; Op1 can be poison
4973 // Op0 = Inst(Op1, NonPoisonOps...)
4974 // ... = Freeze(Op0)
4975 // =>
4976 // Op1 = ...
4977 // Op1.fr = Freeze(Op1)
4978 // ... = Inst(Op1.fr, NonPoisonOps...)
4979
4980 auto CanPushFreeze = [](Value *V) {
4981 if (!isa<Instruction>(V) || isa<PHINode>(V))
4982 return false;
4983
4984 // We can't push the freeze through an instruction which can itself create
4985 // poison. If the only source of new poison is flags, we can simply
4986 // strip them (since we know the only use is the freeze and nothing can
4987 // benefit from them.)
4989 /*ConsiderFlagsAndMetadata*/ false);
4990 };
4991
4992 // Pushing freezes up long instruction chains can be expensive. Instead,
4993 // we directly push the freeze all the way to the leaves. However, we leave
4994 // deduplication of freezes on the same value for freezeOtherUses().
4995 Use *OrigUse = &OrigFI.getOperandUse(0);
4998 Worklist.push_back(OrigUse);
4999 while (!Worklist.empty()) {
5000 auto *U = Worklist.pop_back_val();
5001 Value *V = U->get();
5002 if (!CanPushFreeze(V)) {
5003 // If we can't push through the original instruction, abort the transform.
5004 if (U == OrigUse)
5005 return nullptr;
5006
5007 auto *UserI = cast<Instruction>(U->getUser());
5008 Builder.SetInsertPoint(UserI);
5009 Value *Frozen = Builder.CreateFreeze(V, V->getName() + ".fr");
5010 U->set(Frozen);
5011 continue;
5012 }
5013
5014 auto *I = cast<Instruction>(V);
5015 if (!Visited.insert(I).second)
5016 continue;
5017
5018 // reverse() to emit freezes in a more natural order.
5019 for (Use &Op : reverse(I->operands())) {
5020 Value *OpV = Op.get();
5022 continue;
5023 Worklist.push_back(&Op);
5024 }
5025
5026 I->dropPoisonGeneratingAnnotations();
5027 this->Worklist.add(I);
5028 }
5029
5030 return OrigUse->get();
5031}
5032
5034 PHINode *PN) {
5035 // Detect whether this is a recurrence with a start value and some number of
5036 // backedge values. We'll check whether we can push the freeze through the
5037 // backedge values (possibly dropping poison flags along the way) until we
5038 // reach the phi again. In that case, we can move the freeze to the start
5039 // value.
5040 Use *StartU = nullptr;
5042 for (Use &U : PN->incoming_values()) {
5043 if (DT.dominates(PN->getParent(), PN->getIncomingBlock(U))) {
5044 // Add backedge value to worklist.
5045 Worklist.push_back(U.get());
5046 continue;
5047 }
5048
5049 // Don't bother handling multiple start values.
5050 if (StartU)
5051 return nullptr;
5052 StartU = &U;
5053 }
5054
5055 if (!StartU || Worklist.empty())
5056 return nullptr; // Not a recurrence.
5057
5058 Value *StartV = StartU->get();
5059 BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
5060 bool StartNeedsFreeze = !isGuaranteedNotToBeUndefOrPoison(StartV);
5061 // We can't insert freeze if the start value is the result of the
5062 // terminator (e.g. an invoke).
5063 if (StartNeedsFreeze && StartBB->getTerminator() == StartV)
5064 return nullptr;
5065
5068 while (!Worklist.empty()) {
5069 Value *V = Worklist.pop_back_val();
5070 if (!Visited.insert(V).second)
5071 continue;
5072
5073 if (Visited.size() > 32)
5074 return nullptr; // Limit the total number of values we inspect.
5075
5076 // Assume that PN is non-poison, because it will be after the transform.
5077 if (V == PN || isGuaranteedNotToBeUndefOrPoison(V))
5078 continue;
5079
5082 /*ConsiderFlagsAndMetadata*/ false))
5083 return nullptr;
5084
5085 DropFlags.push_back(I);
5086 append_range(Worklist, I->operands());
5087 }
5088
5089 for (Instruction *I : DropFlags)
5090 I->dropPoisonGeneratingAnnotations();
5091
5092 if (StartNeedsFreeze) {
5093 Builder.SetInsertPoint(StartBB->getTerminator());
5094 Value *FrozenStartV = Builder.CreateFreeze(StartV,
5095 StartV->getName() + ".fr");
5096 replaceUse(*StartU, FrozenStartV);
5097 }
5098 return replaceInstUsesWith(FI, PN);
5099}
5100
5102 Value *Op = FI.getOperand(0);
5103
5104 if (isa<Constant>(Op) || Op->hasOneUse())
5105 return false;
5106
5107 // Move the freeze directly after the definition of its operand, so that
5108 // it dominates the maximum number of uses. Note that it may not dominate
5109 // *all* uses if the operand is an invoke/callbr and the use is in a phi on
5110 // the normal/default destination. This is why the domination check in the
5111 // replacement below is still necessary.
5112 BasicBlock::iterator MoveBefore;
5113 if (isa<Argument>(Op)) {
5114 MoveBefore =
5116 } else {
5117 auto MoveBeforeOpt = cast<Instruction>(Op)->getInsertionPointAfterDef();
5118 if (!MoveBeforeOpt)
5119 return false;
5120 MoveBefore = *MoveBeforeOpt;
5121 }
5122
5123 // Re-point iterator to come after any debug-info records.
5124 MoveBefore.setHeadBit(false);
5125
5126 bool Changed = false;
5127 if (&FI != &*MoveBefore) {
5128 FI.moveBefore(*MoveBefore->getParent(), MoveBefore);
5129 Changed = true;
5130 }
5131
5132 Op->replaceUsesWithIf(&FI, [&](Use &U) -> bool {
5133 bool Dominates = DT.dominates(&FI, U);
5134 Changed |= Dominates;
5135 return Dominates;
5136 });
5137
5138 return Changed;
5139}
5140
5141// Check if any direct or bitcast user of this value is a shuffle instruction.
5143 for (auto *U : V->users()) {
5145 return true;
5146 else if (match(U, m_BitCast(m_Specific(V))) && isUsedWithinShuffleVector(U))
5147 return true;
5148 }
5149 return false;
5150}
5151
5153 Value *Op0 = I.getOperand(0);
5154
5155 if (Value *V = simplifyFreezeInst(Op0, SQ.getWithInstruction(&I)))
5156 return replaceInstUsesWith(I, V);
5157
5158 // freeze (phi const, x) --> phi const, (freeze x)
5159 if (auto *PN = dyn_cast<PHINode>(Op0)) {
5160 if (Instruction *NV = foldOpIntoPhi(I, PN))
5161 return NV;
5162 if (Instruction *NV = foldFreezeIntoRecurrence(I, PN))
5163 return NV;
5164 }
5165
5167 return replaceInstUsesWith(I, NI);
5168
5169 // If I is freeze(undef), check its uses and fold it to a fixed constant.
5170 // - or: pick -1
5171 // - select's condition: if the true value is constant, choose it by making
5172 // the condition true.
5173 // - phi: pick the common constant across operands
5174 // - default: pick 0
5175 //
5176 // Note that this transform is intentionally done here rather than
5177 // via an analysis in InstSimplify or at individual user sites. That is
5178 // because we must produce the same value for all uses of the freeze -
5179 // it's the reason "freeze" exists!
5180 //
5181 // TODO: This could use getBinopAbsorber() / getBinopIdentity() to avoid
5182 // duplicating logic for binops at least.
5183 auto getUndefReplacement = [&](Type *Ty) {
5184 auto pickCommonConstantFromPHI = [](PHINode &PN) -> Value * {
5185 // phi(freeze(undef), C, C). Choose C for freeze so the PHI can be
5186 // removed.
5187 Constant *BestValue = nullptr;
5188 for (Value *V : PN.incoming_values()) {
5189 if (match(V, m_Freeze(m_Undef())))
5190 continue;
5191
5193 if (!C)
5194 return nullptr;
5195
5197 return nullptr;
5198
5199 if (BestValue && BestValue != C)
5200 return nullptr;
5201
5202 BestValue = C;
5203 }
5204 return BestValue;
5205 };
5206
5207 Value *NullValue = Constant::getNullValue(Ty);
5208 Value *BestValue = nullptr;
5209 for (auto *U : I.users()) {
5210 Value *V = NullValue;
5211 if (match(U, m_Or(m_Value(), m_Value())))
5213 else if (match(U, m_Select(m_Specific(&I), m_Constant(), m_Value())))
5214 V = ConstantInt::getTrue(Ty);
5215 else if (match(U, m_c_Select(m_Specific(&I), m_Value(V)))) {
5216 if (V == &I || !isGuaranteedNotToBeUndefOrPoison(V, &AC, &I, &DT))
5217 V = NullValue;
5218 } else if (auto *PHI = dyn_cast<PHINode>(U)) {
5219 if (Value *MaybeV = pickCommonConstantFromPHI(*PHI))
5220 V = MaybeV;
5221 }
5222
5223 if (!BestValue)
5224 BestValue = V;
5225 else if (BestValue != V)
5226 BestValue = NullValue;
5227 }
5228 assert(BestValue && "Must have at least one use");
5229 assert(BestValue != &I && "Cannot replace with itself");
5230 return BestValue;
5231 };
5232
5233 if (match(Op0, m_Undef())) {
5234 // Don't fold freeze(undef/poison) if it's used as a vector operand in
5235 // a shuffle. This may improve codegen for shuffles that allow
5236 // unspecified inputs.
5238 return nullptr;
5239 return replaceInstUsesWith(I, getUndefReplacement(I.getType()));
5240 }
5241
5242 auto getFreezeVectorReplacement = [](Constant *C) -> Constant * {
5243 Type *Ty = C->getType();
5244 auto *VTy = dyn_cast<FixedVectorType>(Ty);
5245 if (!VTy)
5246 return nullptr;
5247 unsigned NumElts = VTy->getNumElements();
5248 Constant *BestValue = Constant::getNullValue(VTy->getScalarType());
5249 for (unsigned i = 0; i != NumElts; ++i) {
5250 Constant *EltC = C->getAggregateElement(i);
5251 if (EltC && !match(EltC, m_Undef())) {
5252 BestValue = EltC;
5253 break;
5254 }
5255 }
5256 return Constant::replaceUndefsWith(C, BestValue);
5257 };
5258
5259 Constant *C;
5260 if (match(Op0, m_Constant(C)) && C->containsUndefOrPoisonElement() &&
5261 !C->containsConstantExpression()) {
5262 if (Constant *Repl = getFreezeVectorReplacement(C))
5263 return replaceInstUsesWith(I, Repl);
5264 }
5265
5266 // Replace uses of Op with freeze(Op).
5267 if (freezeOtherUses(I))
5268 return &I;
5269
5270 return nullptr;
5271}
5272
5273/// Check for case where the call writes to an otherwise dead alloca. This
5274/// shows up for unused out-params in idiomatic C/C++ code. Note that this
5275/// helper *only* analyzes the write; doesn't check any other legality aspect.
5277 auto *CB = dyn_cast<CallBase>(I);
5278 if (!CB)
5279 // TODO: handle e.g. store to alloca here - only worth doing if we extend
5280 // to allow reload along used path as described below. Otherwise, this
5281 // is simply a store to a dead allocation which will be removed.
5282 return false;
5283 std::optional<MemoryLocation> Dest = MemoryLocation::getForDest(CB, TLI);
5284 if (!Dest)
5285 return false;
5286 auto *AI = dyn_cast<AllocaInst>(getUnderlyingObject(Dest->Ptr));
5287 if (!AI)
5288 // TODO: allow malloc?
5289 return false;
5290 // TODO: allow memory access dominated by move point? Note that since AI
5291 // could have a reference to itself captured by the call, we would need to
5292 // account for cycles in doing so.
5293 SmallVector<const User *> AllocaUsers;
5295 auto pushUsers = [&](const Instruction &I) {
5296 for (const User *U : I.users()) {
5297 if (Visited.insert(U).second)
5298 AllocaUsers.push_back(U);
5299 }
5300 };
5301 pushUsers(*AI);
5302 while (!AllocaUsers.empty()) {
5303 auto *UserI = cast<Instruction>(AllocaUsers.pop_back_val());
5304 if (isa<GetElementPtrInst>(UserI) || isa<AddrSpaceCastInst>(UserI)) {
5305 pushUsers(*UserI);
5306 continue;
5307 }
5308 if (UserI == CB)
5309 continue;
5310 // TODO: support lifetime.start/end here
5311 return false;
5312 }
5313 return true;
5314}
5315
5316/// Try to move the specified instruction from its current block into the
5317/// beginning of DestBlock, which can only happen if it's safe to move the
5318/// instruction past all of the instructions between it and the end of its
5319/// block.
5321 BasicBlock *DestBlock) {
5322 BasicBlock *SrcBlock = I->getParent();
5323
5324 // Cannot move control-flow-involving, volatile loads, vaarg, etc.
5325 if (isa<PHINode>(I) || I->isEHPad() || I->mayThrow() || !I->willReturn() ||
5326 I->isTerminator())
5327 return false;
5328
5329 // Do not sink static or dynamic alloca instructions. Static allocas must
5330 // remain in the entry block, and dynamic allocas must not be sunk in between
5331 // a stacksave / stackrestore pair, which would incorrectly shorten its
5332 // lifetime.
5333 if (isa<AllocaInst>(I))
5334 return false;
5335
5336 // Do not sink into catchswitch blocks.
5337 if (isa<CatchSwitchInst>(DestBlock->getTerminator()))
5338 return false;
5339
5340 // Do not sink convergent call instructions.
5341 if (auto *CI = dyn_cast<CallInst>(I)) {
5342 if (CI->isConvergent())
5343 return false;
5344 }
5345
5346 // Unless we can prove that the memory write isn't visibile except on the
5347 // path we're sinking to, we must bail.
5348 if (I->mayWriteToMemory()) {
5349 if (!SoleWriteToDeadLocal(I, TLI))
5350 return false;
5351 }
5352
5353 // We can only sink load instructions if there is nothing between the load and
5354 // the end of block that could change the value.
5355 if (I->mayReadFromMemory() &&
5356 !I->hasMetadata(LLVMContext::MD_invariant_load)) {
5357 // We don't want to do any sophisticated alias analysis, so we only check
5358 // the instructions after I in I's parent block if we try to sink to its
5359 // successor block.
5360 if (DestBlock->getUniquePredecessor() != I->getParent())
5361 return false;
5362 for (BasicBlock::iterator Scan = std::next(I->getIterator()),
5363 E = I->getParent()->end();
5364 Scan != E; ++Scan)
5365 if (Scan->mayWriteToMemory())
5366 return false;
5367 }
5368
5369 I->dropDroppableUses([&](const Use *U) {
5370 auto *I = dyn_cast<Instruction>(U->getUser());
5371 if (I && I->getParent() != DestBlock) {
5372 Worklist.add(I);
5373 return true;
5374 }
5375 return false;
5376 });
5377 /// FIXME: We could remove droppable uses that are not dominated by
5378 /// the new position.
5379
5380 BasicBlock::iterator InsertPos = DestBlock->getFirstInsertionPt();
5381 I->moveBefore(*DestBlock, InsertPos);
5382 ++NumSunkInst;
5383
5384 // Also sink all related debug uses from the source basic block. Otherwise we
5385 // get debug use before the def. Attempt to salvage debug uses first, to
5386 // maximise the range variables have location for. If we cannot salvage, then
5387 // mark the location undef: we know it was supposed to receive a new location
5388 // here, but that computation has been sunk.
5389 SmallVector<DbgVariableRecord *, 2> DbgVariableRecords;
5390 findDbgUsers(I, DbgVariableRecords);
5391 if (!DbgVariableRecords.empty())
5392 tryToSinkInstructionDbgVariableRecords(I, InsertPos, SrcBlock, DestBlock,
5393 DbgVariableRecords);
5394
5395 // PS: there are numerous flaws with this behaviour, not least that right now
5396 // assignments can be re-ordered past other assignments to the same variable
5397 // if they use different Values. Creating more undef assignements can never be
5398 // undone. And salvaging all users outside of this block can un-necessarily
5399 // alter the lifetime of the live-value that the variable refers to.
5400 // Some of these things can be resolved by tolerating debug use-before-defs in
5401 // LLVM-IR, however it depends on the instruction-referencing CodeGen backend
5402 // being used for more architectures.
5403
5404 return true;
5405}
5406
5408 Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
5409 BasicBlock *DestBlock,
5410 SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords) {
5411 // For all debug values in the destination block, the sunk instruction
5412 // will still be available, so they do not need to be dropped.
5413
5414 // Fetch all DbgVariableRecords not already in the destination.
5415 SmallVector<DbgVariableRecord *, 2> DbgVariableRecordsToSalvage;
5416 for (auto &DVR : DbgVariableRecords)
5417 if (DVR->getParent() != DestBlock)
5418 DbgVariableRecordsToSalvage.push_back(DVR);
5419
5420 // Fetch a second collection, of DbgVariableRecords in the source block that
5421 // we're going to sink.
5422 SmallVector<DbgVariableRecord *> DbgVariableRecordsToSink;
5423 for (DbgVariableRecord *DVR : DbgVariableRecordsToSalvage)
5424 if (DVR->getParent() == SrcBlock)
5425 DbgVariableRecordsToSink.push_back(DVR);
5426
5427 // Sort DbgVariableRecords according to their position in the block. This is a
5428 // partial order: DbgVariableRecords attached to different instructions will
5429 // be ordered by the instruction order, but DbgVariableRecords attached to the
5430 // same instruction won't have an order.
5431 auto Order = [](DbgVariableRecord *A, DbgVariableRecord *B) -> bool {
5432 return B->getInstruction()->comesBefore(A->getInstruction());
5433 };
5434 llvm::stable_sort(DbgVariableRecordsToSink, Order);
5435
5436 // If there are two assignments to the same variable attached to the same
5437 // instruction, the ordering between the two assignments is important. Scan
5438 // for this (rare) case and establish which is the last assignment.
5439 using InstVarPair = std::pair<const Instruction *, DebugVariable>;
5441 if (DbgVariableRecordsToSink.size() > 1) {
5443 // Count how many assignments to each variable there is per instruction.
5444 for (DbgVariableRecord *DVR : DbgVariableRecordsToSink) {
5445 DebugVariable DbgUserVariable =
5446 DebugVariable(DVR->getVariable(), DVR->getExpression(),
5447 DVR->getDebugLoc()->getInlinedAt());
5448 CountMap[std::make_pair(DVR->getInstruction(), DbgUserVariable)] += 1;
5449 }
5450
5451 // If there are any instructions with two assignments, add them to the
5452 // FilterOutMap to record that they need extra filtering.
5454 for (auto It : CountMap) {
5455 if (It.second > 1) {
5456 FilterOutMap[It.first] = nullptr;
5457 DupSet.insert(It.first.first);
5458 }
5459 }
5460
5461 // For all instruction/variable pairs needing extra filtering, find the
5462 // latest assignment.
5463 for (const Instruction *Inst : DupSet) {
5464 for (DbgVariableRecord &DVR :
5465 llvm::reverse(filterDbgVars(Inst->getDbgRecordRange()))) {
5466 DebugVariable DbgUserVariable =
5467 DebugVariable(DVR.getVariable(), DVR.getExpression(),
5468 DVR.getDebugLoc()->getInlinedAt());
5469 auto FilterIt =
5470 FilterOutMap.find(std::make_pair(Inst, DbgUserVariable));
5471 if (FilterIt == FilterOutMap.end())
5472 continue;
5473 if (FilterIt->second != nullptr)
5474 continue;
5475 FilterIt->second = &DVR;
5476 }
5477 }
5478 }
5479
5480 // Perform cloning of the DbgVariableRecords that we plan on sinking, filter
5481 // out any duplicate assignments identified above.
5483 SmallSet<DebugVariable, 4> SunkVariables;
5484 for (DbgVariableRecord *DVR : DbgVariableRecordsToSink) {
5486 continue;
5487
5488 DebugVariable DbgUserVariable =
5489 DebugVariable(DVR->getVariable(), DVR->getExpression(),
5490 DVR->getDebugLoc()->getInlinedAt());
5491
5492 // For any variable where there were multiple assignments in the same place,
5493 // ignore all but the last assignment.
5494 if (!FilterOutMap.empty()) {
5495 InstVarPair IVP = std::make_pair(DVR->getInstruction(), DbgUserVariable);
5496 auto It = FilterOutMap.find(IVP);
5497
5498 // Filter out.
5499 if (It != FilterOutMap.end() && It->second != DVR)
5500 continue;
5501 }
5502
5503 if (!SunkVariables.insert(DbgUserVariable).second)
5504 continue;
5505
5506 if (DVR->isDbgAssign())
5507 continue;
5508
5509 DVRClones.emplace_back(DVR->clone());
5510 LLVM_DEBUG(dbgs() << "CLONE: " << *DVRClones.back() << '\n');
5511 }
5512
5513 // Perform salvaging without the clones, then sink the clones.
5514 if (DVRClones.empty())
5515 return;
5516
5517 salvageDebugInfoForDbgValues(*I, DbgVariableRecordsToSalvage);
5518
5519 // The clones are in reverse order of original appearance. Assert that the
5520 // head bit is set on the iterator as we _should_ have received it via
5521 // getFirstInsertionPt. Inserting like this will reverse the clone order as
5522 // we'll repeatedly insert at the head, such as:
5523 // DVR-3 (third insertion goes here)
5524 // DVR-2 (second insertion goes here)
5525 // DVR-1 (first insertion goes here)
5526 // Any-Prior-DVRs
5527 // InsertPtInst
5528 assert(InsertPos.getHeadBit());
5529 for (DbgVariableRecord *DVRClone : DVRClones) {
5530 InsertPos->getParent()->insertDbgRecordBefore(DVRClone, InsertPos);
5531 LLVM_DEBUG(dbgs() << "SINK: " << *DVRClone << '\n');
5532 }
5533}
5534
5536 while (!Worklist.isEmpty()) {
5537 // Walk deferred instructions in reverse order, and push them to the
5538 // worklist, which means they'll end up popped from the worklist in-order.
5539 while (Instruction *I = Worklist.popDeferred()) {
5540 // Check to see if we can DCE the instruction. We do this already here to
5541 // reduce the number of uses and thus allow other folds to trigger.
5542 // Note that eraseInstFromFunction() may push additional instructions on
5543 // the deferred worklist, so this will DCE whole instruction chains.
5546 ++NumDeadInst;
5547 continue;
5548 }
5549
5550 Worklist.push(I);
5551 }
5552
5553 Instruction *I = Worklist.removeOne();
5554 if (I == nullptr) continue; // skip null values.
5555
5556 // Check to see if we can DCE the instruction.
5559 ++NumDeadInst;
5560 continue;
5561 }
5562
5563 if (!DebugCounter::shouldExecute(VisitCounter))
5564 continue;
5565
5566 // See if we can trivially sink this instruction to its user if we can
5567 // prove that the successor is not executed more frequently than our block.
5568 // Return the UserBlock if successful.
5569 auto getOptionalSinkBlockForInst =
5570 [this](Instruction *I) -> std::optional<BasicBlock *> {
5571 if (!EnableCodeSinking)
5572 return std::nullopt;
5573
5574 BasicBlock *BB = I->getParent();
5575 BasicBlock *UserParent = nullptr;
5576 unsigned NumUsers = 0;
5577
5578 for (Use &U : I->uses()) {
5579 User *User = U.getUser();
5580 if (User->isDroppable())
5581 continue;
5582 if (NumUsers > MaxSinkNumUsers)
5583 return std::nullopt;
5584
5585 Instruction *UserInst = cast<Instruction>(User);
5586 // Special handling for Phi nodes - get the block the use occurs in.
5587 BasicBlock *UserBB = UserInst->getParent();
5588 if (PHINode *PN = dyn_cast<PHINode>(UserInst))
5589 UserBB = PN->getIncomingBlock(U);
5590 // Bail out if we have uses in different blocks. We don't do any
5591 // sophisticated analysis (i.e finding NearestCommonDominator of these
5592 // use blocks).
5593 if (UserParent && UserParent != UserBB)
5594 return std::nullopt;
5595 UserParent = UserBB;
5596
5597 // Make sure these checks are done only once, naturally we do the checks
5598 // the first time we get the userparent, this will save compile time.
5599 if (NumUsers == 0) {
5600 // Try sinking to another block. If that block is unreachable, then do
5601 // not bother. SimplifyCFG should handle it.
5602 if (UserParent == BB || !DT.isReachableFromEntry(UserParent))
5603 return std::nullopt;
5604
5605 auto *Term = UserParent->getTerminator();
5606 // See if the user is one of our successors that has only one
5607 // predecessor, so that we don't have to split the critical edge.
5608 // Another option where we can sink is a block that ends with a
5609 // terminator that does not pass control to other block (such as
5610 // return or unreachable or resume). In this case:
5611 // - I dominates the User (by SSA form);
5612 // - the User will be executed at most once.
5613 // So sinking I down to User is always profitable or neutral.
5614 if (UserParent->getUniquePredecessor() != BB && !succ_empty(Term))
5615 return std::nullopt;
5616
5617 assert(DT.dominates(BB, UserParent) && "Dominance relation broken?");
5618 }
5619
5620 NumUsers++;
5621 }
5622
5623 // No user or only has droppable users.
5624 if (!UserParent)
5625 return std::nullopt;
5626
5627 return UserParent;
5628 };
5629
5630 auto OptBB = getOptionalSinkBlockForInst(I);
5631 if (OptBB) {
5632 auto *UserParent = *OptBB;
5633 // Okay, the CFG is simple enough, try to sink this instruction.
5634 if (tryToSinkInstruction(I, UserParent)) {
5635 LLVM_DEBUG(dbgs() << "IC: Sink: " << *I << '\n');
5636 MadeIRChange = true;
5637 // We'll add uses of the sunk instruction below, but since
5638 // sinking can expose opportunities for it's *operands* add
5639 // them to the worklist
5640 for (Use &U : I->operands())
5641 if (Instruction *OpI = dyn_cast<Instruction>(U.get()))
5642 Worklist.push(OpI);
5643 }
5644 }
5645
5646 // Now that we have an instruction, try combining it to simplify it.
5647 Builder.SetInsertPoint(I);
5648 Builder.CollectMetadataToCopy(
5649 I, {LLVMContext::MD_dbg, LLVMContext::MD_annotation});
5650
5651#ifndef NDEBUG
5652 std::string OrigI;
5653#endif
5654 LLVM_DEBUG(raw_string_ostream SS(OrigI); I->print(SS););
5655 LLVM_DEBUG(dbgs() << "IC: Visiting: " << OrigI << '\n');
5656
5657 if (Instruction *Result = visit(*I)) {
5658 ++NumCombined;
5659 // Should we replace the old instruction with a new one?
5660 if (Result != I) {
5661 LLVM_DEBUG(dbgs() << "IC: Old = " << *I << '\n'
5662 << " New = " << *Result << '\n');
5663
5664 // We copy the old instruction's DebugLoc to the new instruction, unless
5665 // InstCombine already assigned a DebugLoc to it, in which case we
5666 // should trust the more specifically selected DebugLoc.
5667 Result->setDebugLoc(Result->getDebugLoc().orElse(I->getDebugLoc()));
5668 // We also copy annotation metadata to the new instruction.
5669 Result->copyMetadata(*I, LLVMContext::MD_annotation);
5670 // Everything uses the new instruction now.
5671 I->replaceAllUsesWith(Result);
5672
5673 // Move the name to the new instruction first.
5674 Result->takeName(I);
5675
5676 // Insert the new instruction into the basic block...
5677 BasicBlock *InstParent = I->getParent();
5678 BasicBlock::iterator InsertPos = I->getIterator();
5679
5680 // Are we replace a PHI with something that isn't a PHI, or vice versa?
5681 if (isa<PHINode>(Result) != isa<PHINode>(I)) {
5682 // We need to fix up the insertion point.
5683 if (isa<PHINode>(I)) // PHI -> Non-PHI
5684 InsertPos = InstParent->getFirstInsertionPt();
5685 else // Non-PHI -> PHI
5686 InsertPos = InstParent->getFirstNonPHIIt();
5687 }
5688
5689 Result->insertInto(InstParent, InsertPos);
5690
5691 // Push the new instruction and any users onto the worklist.
5692 Worklist.pushUsersToWorkList(*Result);
5693 Worklist.push(Result);
5694
5696 } else {
5697 LLVM_DEBUG(dbgs() << "IC: Mod = " << OrigI << '\n'
5698 << " New = " << *I << '\n');
5699
5700 // If the instruction was modified, it's possible that it is now dead.
5701 // if so, remove it.
5704 } else {
5705 Worklist.pushUsersToWorkList(*I);
5706 Worklist.push(I);
5707 }
5708 }
5709 MadeIRChange = true;
5710 }
5711 }
5712
5713 Worklist.zap();
5714 return MadeIRChange;
5715}
5716
5717// Track the scopes used by !alias.scope and !noalias. In a function, a
5718// @llvm.experimental.noalias.scope.decl is only useful if that scope is used
5719// by both sets. If not, the declaration of the scope can be safely omitted.
5720// The MDNode of the scope can be omitted as well for the instructions that are
5721// part of this function. We do not do that at this point, as this might become
5722// too time consuming to do.
5724 SmallPtrSet<const MDNode *, 8> UsedAliasScopesAndLists;
5725 SmallPtrSet<const MDNode *, 8> UsedNoAliasScopesAndLists;
5726
5727public:
5729 // This seems to be faster than checking 'mayReadOrWriteMemory()'.
5730 if (!I->hasMetadataOtherThanDebugLoc())
5731 return;
5732
5733 auto Track = [](Metadata *ScopeList, auto &Container) {
5734 const auto *MDScopeList = dyn_cast_or_null<MDNode>(ScopeList);
5735 if (!MDScopeList || !Container.insert(MDScopeList).second)
5736 return;
5737 for (const auto &MDOperand : MDScopeList->operands())
5738 if (auto *MDScope = dyn_cast<MDNode>(MDOperand))
5739 Container.insert(MDScope);
5740 };
5741
5742 Track(I->getMetadata(LLVMContext::MD_alias_scope), UsedAliasScopesAndLists);
5743 Track(I->getMetadata(LLVMContext::MD_noalias), UsedNoAliasScopesAndLists);
5744 }
5745
5748 if (!Decl)
5749 return false;
5750
5751 assert(Decl->use_empty() &&
5752 "llvm.experimental.noalias.scope.decl in use ?");
5753 const MDNode *MDSL = Decl->getScopeList();
5754 assert(MDSL->getNumOperands() == 1 &&
5755 "llvm.experimental.noalias.scope should refer to a single scope");
5756 auto &MDOperand = MDSL->getOperand(0);
5757 if (auto *MD = dyn_cast<MDNode>(MDOperand))
5758 return !UsedAliasScopesAndLists.contains(MD) ||
5759 !UsedNoAliasScopesAndLists.contains(MD);
5760
5761 // Not an MDNode ? throw away.
5762 return true;
5763 }
5764};
5765
5766/// Populate the IC worklist from a function, by walking it in reverse
5767/// post-order and adding all reachable code to the worklist.
5768///
5769/// This has a couple of tricks to make the code faster and more powerful. In
5770/// particular, we constant fold and DCE instructions as we go, to avoid adding
5771/// them to the worklist (this significantly speeds up instcombine on code where
5772/// many instructions are dead or constant). Additionally, if we find a branch
5773/// whose condition is a known constant, we only visit the reachable successors.
5775 bool MadeIRChange = false;
5777 SmallVector<Instruction *, 128> InstrsForInstructionWorklist;
5778 DenseMap<Constant *, Constant *> FoldedConstants;
5779 AliasScopeTracker SeenAliasScopes;
5780
5781 auto HandleOnlyLiveSuccessor = [&](BasicBlock *BB, BasicBlock *LiveSucc) {
5782 for (BasicBlock *Succ : successors(BB))
5783 if (Succ != LiveSucc && DeadEdges.insert({BB, Succ}).second)
5784 for (PHINode &PN : Succ->phis())
5785 for (Use &U : PN.incoming_values())
5786 if (PN.getIncomingBlock(U) == BB && !isa<PoisonValue>(U)) {
5787 U.set(PoisonValue::get(PN.getType()));
5788 MadeIRChange = true;
5789 }
5790 };
5791
5792 for (BasicBlock *BB : RPOT) {
5793 if (!BB->isEntryBlock() && all_of(predecessors(BB), [&](BasicBlock *Pred) {
5794 return DeadEdges.contains({Pred, BB}) || DT.dominates(BB, Pred);
5795 })) {
5796 HandleOnlyLiveSuccessor(BB, nullptr);
5797 continue;
5798 }
5799 LiveBlocks.insert(BB);
5800
5801 for (Instruction &Inst : llvm::make_early_inc_range(*BB)) {
5802 // ConstantProp instruction if trivially constant.
5803 if (!Inst.use_empty() &&
5804 (Inst.getNumOperands() == 0 || isa<Constant>(Inst.getOperand(0))))
5805 if (Constant *C = ConstantFoldInstruction(&Inst, DL, &TLI)) {
5806 LLVM_DEBUG(dbgs() << "IC: ConstFold to: " << *C << " from: " << Inst
5807 << '\n');
5808 Inst.replaceAllUsesWith(C);
5809 ++NumConstProp;
5810 if (isInstructionTriviallyDead(&Inst, &TLI))
5811 Inst.eraseFromParent();
5812 MadeIRChange = true;
5813 continue;
5814 }
5815
5816 // See if we can constant fold its operands.
5817 for (Use &U : Inst.operands()) {
5819 continue;
5820
5821 auto *C = cast<Constant>(U);
5822 Constant *&FoldRes = FoldedConstants[C];
5823 if (!FoldRes)
5824 FoldRes = ConstantFoldConstant(C, DL, &TLI);
5825
5826 if (FoldRes != C) {
5827 LLVM_DEBUG(dbgs() << "IC: ConstFold operand of: " << Inst
5828 << "\n Old = " << *C
5829 << "\n New = " << *FoldRes << '\n');
5830 U = FoldRes;
5831 MadeIRChange = true;
5832 }
5833 }
5834
5835 // Skip processing debug and pseudo intrinsics in InstCombine. Processing
5836 // these call instructions consumes non-trivial amount of time and
5837 // provides no value for the optimization.
5838 if (!Inst.isDebugOrPseudoInst()) {
5839 InstrsForInstructionWorklist.push_back(&Inst);
5840 SeenAliasScopes.analyse(&Inst);
5841 }
5842 }
5843
5844 // If this is a branch or switch on a constant, mark only the single
5845 // live successor. Otherwise assume all successors are live.
5846 Instruction *TI = BB->getTerminator();
5847 if (BranchInst *BI = dyn_cast<BranchInst>(TI); BI && BI->isConditional()) {
5848 if (isa<UndefValue>(BI->getCondition())) {
5849 // Branch on undef is UB.
5850 HandleOnlyLiveSuccessor(BB, nullptr);
5851 continue;
5852 }
5853 if (auto *Cond = dyn_cast<ConstantInt>(BI->getCondition())) {
5854 bool CondVal = Cond->getZExtValue();
5855 HandleOnlyLiveSuccessor(BB, BI->getSuccessor(!CondVal));
5856 continue;
5857 }
5858 } else if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
5859 if (isa<UndefValue>(SI->getCondition())) {
5860 // Switch on undef is UB.
5861 HandleOnlyLiveSuccessor(BB, nullptr);
5862 continue;
5863 }
5864 if (auto *Cond = dyn_cast<ConstantInt>(SI->getCondition())) {
5865 HandleOnlyLiveSuccessor(BB,
5866 SI->findCaseValue(Cond)->getCaseSuccessor());
5867 continue;
5868 }
5869 }
5870 }
5871
5872 // Remove instructions inside unreachable blocks. This prevents the
5873 // instcombine code from having to deal with some bad special cases, and
5874 // reduces use counts of instructions.
5875 for (BasicBlock &BB : F) {
5876 if (LiveBlocks.count(&BB))
5877 continue;
5878
5879 unsigned NumDeadInstInBB;
5880 NumDeadInstInBB = removeAllNonTerminatorAndEHPadInstructions(&BB);
5881
5882 MadeIRChange |= NumDeadInstInBB != 0;
5883 NumDeadInst += NumDeadInstInBB;
5884 }
5885
5886 // Once we've found all of the instructions to add to instcombine's worklist,
5887 // add them in reverse order. This way instcombine will visit from the top
5888 // of the function down. This jives well with the way that it adds all uses
5889 // of instructions to the worklist after doing a transformation, thus avoiding
5890 // some N^2 behavior in pathological cases.
5891 Worklist.reserve(InstrsForInstructionWorklist.size());
5892 for (Instruction *Inst : reverse(InstrsForInstructionWorklist)) {
5893 // DCE instruction if trivially dead. As we iterate in reverse program
5894 // order here, we will clean up whole chains of dead instructions.
5895 if (isInstructionTriviallyDead(Inst, &TLI) ||
5896 SeenAliasScopes.isNoAliasScopeDeclDead(Inst)) {
5897 ++NumDeadInst;
5898 LLVM_DEBUG(dbgs() << "IC: DCE: " << *Inst << '\n');
5899 salvageDebugInfo(*Inst);
5900 Inst->eraseFromParent();
5901 MadeIRChange = true;
5902 continue;
5903 }
5904
5905 Worklist.push(Inst);
5906 }
5907
5908 return MadeIRChange;
5909}
5910
5912 // Collect backedges.
5914 for (BasicBlock *BB : RPOT) {
5915 Visited.insert(BB);
5916 for (BasicBlock *Succ : successors(BB))
5917 if (Visited.contains(Succ))
5918 BackEdges.insert({BB, Succ});
5919 }
5920 ComputedBackEdges = true;
5921}
5922
5928 const InstCombineOptions &Opts) {
5929 auto &DL = F.getDataLayout();
5930 bool VerifyFixpoint = Opts.VerifyFixpoint &&
5931 !F.hasFnAttribute("instcombine-no-verify-fixpoint");
5932
5933 /// Builder - This is an IRBuilder that automatically inserts new
5934 /// instructions into the worklist when they are created.
5936 F.getContext(), TargetFolder(DL),
5937 IRBuilderCallbackInserter([&Worklist, &AC](Instruction *I) {
5938 Worklist.add(I);
5939 if (auto *Assume = dyn_cast<AssumeInst>(I))
5940 AC.registerAssumption(Assume);
5941 }));
5942
5944
5945 // Lower dbg.declare intrinsics otherwise their value may be clobbered
5946 // by instcombiner.
5947 bool MadeIRChange = false;
5949 MadeIRChange = LowerDbgDeclare(F);
5950
5951 // Iterate while there is work to do.
5952 unsigned Iteration = 0;
5953 while (true) {
5954 if (Iteration >= Opts.MaxIterations && !VerifyFixpoint) {
5955 LLVM_DEBUG(dbgs() << "\n\n[IC] Iteration limit #" << Opts.MaxIterations
5956 << " on " << F.getName()
5957 << " reached; stopping without verifying fixpoint\n");
5958 break;
5959 }
5960
5961 ++Iteration;
5962 ++NumWorklistIterations;
5963 LLVM_DEBUG(dbgs() << "\n\nINSTCOMBINE ITERATION #" << Iteration << " on "
5964 << F.getName() << "\n");
5965
5966 InstCombinerImpl IC(Worklist, Builder, F, AA, AC, TLI, TTI, DT, ORE, BFI,
5967 BPI, PSI, DL, RPOT);
5969 bool MadeChangeInThisIteration = IC.prepareWorklist(F);
5970 MadeChangeInThisIteration |= IC.run();
5971 if (!MadeChangeInThisIteration)
5972 break;
5973
5974 MadeIRChange = true;
5975 if (Iteration > Opts.MaxIterations) {
5977 "Instruction Combining on " + Twine(F.getName()) +
5978 " did not reach a fixpoint after " + Twine(Opts.MaxIterations) +
5979 " iterations. " +
5980 "Use 'instcombine<no-verify-fixpoint>' or function attribute "
5981 "'instcombine-no-verify-fixpoint' to suppress this error.");
5982 }
5983 }
5984
5985 if (Iteration == 1)
5986 ++NumOneIteration;
5987 else if (Iteration == 2)
5988 ++NumTwoIterations;
5989 else if (Iteration == 3)
5990 ++NumThreeIterations;
5991 else
5992 ++NumFourOrMoreIterations;
5993
5994 return MadeIRChange;
5995}
5996
5998
6000 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
6001 static_cast<PassInfoMixin<InstCombinePass> *>(this)->printPipeline(
6002 OS, MapClassName2PassName);
6003 OS << '<';
6004 OS << "max-iterations=" << Options.MaxIterations << ";";
6005 OS << (Options.VerifyFixpoint ? "" : "no-") << "verify-fixpoint";
6006 OS << '>';
6007}
6008
6009char InstCombinePass::ID = 0;
6010
6013 auto &LRT = AM.getResult<LastRunTrackingAnalysis>(F);
6014 // No changes since last InstCombine pass, exit early.
6015 if (LRT.shouldSkip(&ID))
6016 return PreservedAnalyses::all();
6017
6018 auto &AC = AM.getResult<AssumptionAnalysis>(F);
6019 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
6020 auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
6022 auto &TTI = AM.getResult<TargetIRAnalysis>(F);
6023
6024 auto *AA = &AM.getResult<AAManager>(F);
6025 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
6026 ProfileSummaryInfo *PSI =
6027 MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
6028 auto *BFI = (PSI && PSI->hasProfileSummary()) ?
6029 &AM.getResult<BlockFrequencyAnalysis>(F) : nullptr;
6031
6032 if (!combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE,
6033 BFI, BPI, PSI, Options)) {
6034 // No changes, all analyses are preserved.
6035 LRT.update(&ID, /*Changed=*/false);
6036 return PreservedAnalyses::all();
6037 }
6038
6039 // Mark all the analyses that instcombine updates as preserved.
6041 LRT.update(&ID, /*Changed=*/true);
6044 return PA;
6045}
6046
6062
6064 if (skipFunction(F))
6065 return false;
6066
6067 // Required analyses.
6068 auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
6069 auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
6070 auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
6072 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
6074
6075 // Optional analyses.
6076 ProfileSummaryInfo *PSI =
6078 BlockFrequencyInfo *BFI =
6079 (PSI && PSI->hasProfileSummary()) ?
6081 nullptr;
6082 BranchProbabilityInfo *BPI = nullptr;
6083 if (auto *WrapperPass =
6085 BPI = &WrapperPass->getBPI();
6086
6087 return combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE,
6088 BFI, BPI, PSI, InstCombineOptions());
6089}
6090
6092
6096
6098 "Combine redundant instructions", false, false)
6109 "Combine redundant instructions", false, false)
6110
6111// Initialization Routines
6115
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
Rewrite undef for PHI
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This is the interface for LLVM's primary stateless and local alias analysis.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool willNotOverflow(BinaryOpIntrinsic *BO, LazyValueInfo *LVI)
DXIL Resource Access
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
This file defines the DenseMap class.
static bool isSigned(unsigned int Opcode)
This is the interface for a simple mod/ref and alias analysis over globals.
Hexagon Common GEP
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This header defines various interfaces for pass management in LLVM.
This defines the Use class.
iv Induction Variable Users
Definition IVUsers.cpp:48
static bool leftDistributesOverRight(Instruction::BinaryOps LOp, bool HasNUW, bool HasNSW, Intrinsic::ID ROp)
Return whether "X LOp (Y ROp Z)" is always equal to "(X LOp Y) ROp (X LOp Z)".
This file provides internal interfaces used to implement the InstCombine.
This file provides the primary interface to the instcombine pass.
static Value * simplifySwitchOnSelectUsingRanges(SwitchInst &SI, SelectInst *Select, bool IsTrueArm)
static bool isUsedWithinShuffleVector(Value *V)
static bool isNeverEqualToUnescapedAlloc(Value *V, const TargetLibraryInfo &TLI, Instruction *AI)
static bool shorter_filter(const Value *LHS, const Value *RHS)
static Instruction * combineConstantOffsets(GetElementPtrInst &GEP, InstCombinerImpl &IC)
Combine constant offsets separated by variable offsets.
static Instruction * foldSelectGEP(GetElementPtrInst &GEP, InstCombiner::BuilderTy &Builder)
Thread a GEP operation with constant indices through the constant true/false arms of a select.
static bool shouldMergeGEPs(GEPOperator &GEP, GEPOperator &Src)
static cl::opt< unsigned > MaxArraySize("instcombine-maxarray-size", cl::init(1024), cl::desc("Maximum array size considered when doing a combine"))
static cl::opt< unsigned > ShouldLowerDbgDeclare("instcombine-lower-dbg-declare", cl::Hidden, cl::init(true))
static bool hasNoSignedWrap(BinaryOperator &I)
static bool simplifyAssocCastAssoc(BinaryOperator *BinOp1, InstCombinerImpl &IC)
Combine constant operands of associative operations either before or after a cast to eliminate one of...
static bool combineInstructionsOverFunction(Function &F, InstructionWorklist &Worklist, AliasAnalysis *AA, AssumptionCache &AC, TargetLibraryInfo &TLI, TargetTransformInfo &TTI, DominatorTree &DT, OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI, BranchProbabilityInfo *BPI, ProfileSummaryInfo *PSI, const InstCombineOptions &Opts)
static Value * simplifyInstructionWithPHI(Instruction &I, PHINode *PN, Value *InValue, BasicBlock *InBB, const DataLayout &DL, const SimplifyQuery SQ)
static bool shouldCanonicalizeGEPToPtrAdd(GetElementPtrInst &GEP)
Return true if we should canonicalize the gep to an i8 ptradd.
static void ClearSubclassDataAfterReassociation(BinaryOperator &I)
Conservatively clears subclassOptionalData after a reassociation or commutation.
static Value * getIdentityValue(Instruction::BinaryOps Opcode, Value *V)
This function returns identity value for given opcode, which can be used to factor patterns like (X *...
static Value * foldFrexpOfSelect(ExtractValueInst &EV, IntrinsicInst *FrexpCall, SelectInst *SelectInst, InstCombiner::BuilderTy &Builder)
static std::optional< std::pair< Value *, Value * > > matchSymmetricPhiNodesPair(PHINode *LHS, PHINode *RHS)
static Value * foldOperationIntoSelectOperand(Instruction &I, SelectInst *SI, Value *NewOp, InstCombiner &IC)
static Instruction * canonicalizeGEPOfConstGEPI8(GetElementPtrInst &GEP, GEPOperator *Src, InstCombinerImpl &IC)
static Instruction * tryToMoveFreeBeforeNullTest(CallInst &FI, const DataLayout &DL)
Move the call to free before a NULL test.
static Value * simplifyOperationIntoSelectOperand(Instruction &I, SelectInst *SI, bool IsTrueArm)
static bool rightDistributesOverLeft(Instruction::BinaryOps LOp, Instruction::BinaryOps ROp)
Return whether "(X LOp Y) ROp Z" is always equal to "(X ROp Z) LOp (Y ROp Z)".
static Value * tryFactorization(BinaryOperator &I, const SimplifyQuery &SQ, InstCombiner::BuilderTy &Builder, Instruction::BinaryOps InnerOpcode, Value *A, Value *B, Value *C, Value *D)
This tries to simplify binary operations by factorizing out common terms (e.
static bool isRemovableWrite(CallBase &CB, Value *UsedV, const TargetLibraryInfo &TLI)
Given a call CB which uses an address UsedV, return true if we can prove the call's only possible eff...
static Instruction::BinaryOps getBinOpsForFactorization(Instruction::BinaryOps TopOpcode, BinaryOperator *Op, Value *&LHS, Value *&RHS, BinaryOperator *OtherOp)
This function predicates factorization using distributive laws.
static bool hasNoUnsignedWrap(BinaryOperator &I)
static bool SoleWriteToDeadLocal(Instruction *I, TargetLibraryInfo &TLI)
Check for case where the call writes to an otherwise dead alloca.
static cl::opt< unsigned > MaxSinkNumUsers("instcombine-max-sink-users", cl::init(32), cl::desc("Maximum number of undroppable users for instruction sinking"))
static Instruction * foldGEPOfPhi(GetElementPtrInst &GEP, PHINode *PN, IRBuilderBase &Builder)
static std::optional< ModRefInfo > isAllocSiteRemovable(Instruction *AI, SmallVectorImpl< WeakTrackingVH > &Users, const TargetLibraryInfo &TLI, bool KnowInit)
static bool isCatchAll(EHPersonality Personality, Constant *TypeInfo)
Return 'true' if the given typeinfo will match anything.
static cl::opt< bool > EnableCodeSinking("instcombine-code-sinking", cl::desc("Enable code sinking"), cl::init(true))
static bool maintainNoSignedWrap(BinaryOperator &I, Value *B, Value *C)
static GEPNoWrapFlags getMergedGEPNoWrapFlags(GEPOperator &GEP1, GEPOperator &GEP2)
Determine nowrap flags for (gep (gep p, x), y) to (gep p, (x + y)) transform.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
This file contains the declarations for metadata subclasses.
#define T
uint64_t IntrinsicInst * II
static bool IsSelect(MachineInstr &MI)
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
const SmallVectorImpl< MachineOperand > & Cond
static unsigned getNumElements(Type *Ty)
unsigned OpIndex
BaseType
A given derived pointer can have multiple base pointers through phi/selects.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:114
static unsigned getScalarSizeInBits(Type *Ty)
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
This pass exposes codegen information to IR-level passes.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition VPlanSLP.cpp:247
Value * RHS
Value * LHS
static const uint32_t IV[8]
Definition blake3_impl.h:83
bool isNoAliasScopeDeclDead(Instruction *Inst)
void analyse(Instruction *I)
A manager for alias analyses.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:344
static LLVM_ABI unsigned int semanticsPrecision(const fltSemantics &)
Definition APFloat.cpp:296
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:234
static LLVM_ABI void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
Definition APInt.cpp:1758
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition APInt.h:423
static LLVM_ABI void sdivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Definition APInt.cpp:1890
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:936
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:371
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:380
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1488
LLVM_ABI APInt sadd_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1928
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition APInt.h:827
LLVM_ABI APInt smul_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1960
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition APInt.h:334
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition APInt.h:1150
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:440
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:306
LLVM_ABI APInt ssub_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1941
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition APInt.h:851
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition Pass.cpp:270
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
Definition ArrayRef.h:224
size_t size() const
size - Get the array size.
Definition ArrayRef.h:147
Class to represent array types.
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
uint64_t getNumElements() const
Type * getElementType() const
A function analysis which provides an AssumptionCache.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
LLVM_ABI void registerAssumption(AssumeInst *CI)
Add an @llvm.assume intrinsic to this function's cache.
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:69
LLVM_ABI uint64_t getDereferenceableBytes() const
Returns the number of dereferenceable bytes from the dereferenceable attribute.
bool isValid() const
Return true if the attribute is any kind of attribute.
Definition Attributes.h:223
Legacy wrapper pass to provide the BasicAAResult object.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition BasicBlock.h:528
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
LLVM_ABI iterator_range< filter_iterator< BasicBlock::const_iterator, std::function< bool(const Instruction &)> > > instructionsWithoutDebug(bool SkipPseudoOp=true) const
Return a const iterator range over the instructions in the block, skipping any debug instructions.
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
LLVM_ABI bool isEntryBlock() const
Return true if this is the entry block of the containing function.
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction & front() const
Definition BasicBlock.h:482
LLVM_ABI const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
LLVM_ABI const_iterator getFirstNonPHIOrDbgOrAlloca() const
Returns an iterator to the first instruction in this block that is not a PHINode, a debug intrinsic,...
size_t size() const
Definition BasicBlock.h:480
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition BasicBlock.h:233
static LLVM_ABI BinaryOperator * CreateNeg(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Helper functions to construct and inspect unary operations (NEG and NOT) via binary operators SUB and...
BinaryOps getOpcode() const
Definition InstrTypes.h:374
static LLVM_ABI BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
static BinaryOperator * CreateNUW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name="")
Definition InstrTypes.h:294
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Conditional or Unconditional Branch instruction.
LLVM_ABI void swapSuccessors()
Swap the successors of this branch instruction.
bool isConditional() const
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
Analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
void setAttributes(AttributeList A)
Set the attributes for this call.
bool doesNotThrow() const
Determine if the call cannot unwind.
Value * getArgOperand(unsigned i) const
AttributeList getAttributes() const
Return the attributes for this call.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:676
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:699
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:701
@ ICMP_NE
not equal
Definition InstrTypes.h:698
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:827
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:789
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
ConstantArray - Constant Array Declarations.
Definition Constants.h:433
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
A vector constant whose element type is a simple 1/2/4/8-byte integer or float/double,...
Definition Constants.h:776
static LLVM_ABI Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getNot(Constant *C)
static LLVM_ABI Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getBinOpIdentity(unsigned Opcode, Type *Ty, bool AllowRHSConstant=false, bool NSZ=false)
Return the identity constant for a binary opcode.
static LLVM_ABI Constant * getNeg(Constant *C, bool HasNSW=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
static LLVM_ABI ConstantInt * getBool(LLVMContext &Context, bool V)
This class represents a range of values.
LLVM_ABI bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const
Set up Pred and RHS such that ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.
static LLVM_ABI ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
LLVM_ABI bool contains(const APInt &Val) const
Return true if the specified value is in the set.
static LLVM_ABI ConstantRange makeExactNoWrapRegion(Instruction::BinaryOps BinOp, const APInt &Other, unsigned NoWrapKind)
Produce the range that contains X if and only if "X BinOp Other" does not wrap.
Constant Vector Declarations.
Definition Constants.h:517
static LLVM_ABI Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
Definition Constant.h:43
static LLVM_ABI Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
static LLVM_ABI Constant * replaceUndefsWith(Constant *C, Constant *Replacement)
Try to replace undefined constant C or undefined elements in C with Replacement.
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
const Constant * stripPointerCasts() const
Definition Constant.h:219
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
LLVM_ABI Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
LLVM_ABI bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition Constants.cpp:90
static LLVM_ABI DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:63
Record of a variable value-assignment, aka a non instruction representation of the dbg....
static bool shouldExecute(unsigned CounterName)
Identifies a unique instance of a variable.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition DenseMap.h:194
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:167
bool empty() const
Definition DenseMap.h:109
iterator end()
Definition DenseMap.h:81
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:222
Analysis pass which computes a DominatorTree.
Definition Dominators.h:284
Legacy analysis pass which computes a DominatorTree.
Definition Dominators.h:322
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:165
This instruction extracts a struct member or array element value from an aggregate value.
ArrayRef< unsigned > getIndices() const
iterator_range< idx_iterator > indices() const
idx_iterator idx_end() const
static ExtractValueInst * Create(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
idx_iterator idx_begin() const
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition Operator.h:200
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:22
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:314
FunctionPass(char &pid)
Definition Pass.h:316
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition Pass.cpp:188
const BasicBlock & getEntryBlock() const
Definition Function.h:807
Represents flags for the getelementptr instruction/expression.
static GEPNoWrapFlags inBounds()
static GEPNoWrapFlags all()
static GEPNoWrapFlags noUnsignedWrap()
GEPNoWrapFlags intersectForReassociate(GEPNoWrapFlags Other) const
Given (gep (gep p, x), y), determine the nowrap flags for (gep (gep, p, y), x).
bool hasNoUnsignedWrap() const
bool isInBounds() const
GEPNoWrapFlags intersectForOffsetAdd(GEPNoWrapFlags Other) const
Given (gep (gep p, x), y), determine the nowrap flags for (gep p, x+y).
static GEPNoWrapFlags none()
GEPNoWrapFlags getNoWrapFlags() const
Definition Operator.h:425
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static LLVM_ABI Type * getTypeAtIndex(Type *Ty, Value *Idx)
Return the type of the element at the given index of an indexable type.
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
static LLVM_ABI Type * getIndexedType(Type *Ty, ArrayRef< Value * > IdxList)
Returns the result type of a getelementptr with the given source element type and indexes.
static GetElementPtrInst * CreateInBounds(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Create an "inbounds" getelementptr.
Legacy wrapper pass to provide the GlobalsAAResult object.
This instruction compares its operands according to the predicate given to the constructor.
CmpPredicate getCmpPredicate() const
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
Common base class shared among various IRBuilders.
Definition IRBuilder.h:114
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Definition IRBuilder.h:2039
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
Definition IRBuilder.h:538
Provides an 'InsertHelper' that calls a user-provided callback after performing the default insertion...
Definition IRBuilder.h:75
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2788
This instruction inserts a struct field of array element value into an aggregate value.
static InsertValueInst * Create(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
LLVM_ABI InstCombinePass(InstCombineOptions Opts={})
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false)
Given an instruction with a select as one operand and a constant as the other operand,...
Instruction * foldBinOpOfSelectAndCastOfSelectCondition(BinaryOperator &I)
Tries to simplify binops of select and cast of the select condition.
Instruction * foldBinOpIntoSelectOrPhi(BinaryOperator &I)
This is a convenience wrapper function for the above two functions.
bool SimplifyAssociativeOrCommutative(BinaryOperator &I)
Performs a few simplifications for operators which are associative or commutative.
Instruction * visitGEPOfGEP(GetElementPtrInst &GEP, GEPOperator *Src)
Value * foldUsingDistributiveLaws(BinaryOperator &I)
Tries to simplify binary operations which some other binary operation distributes over.
Instruction * foldBinOpShiftWithShift(BinaryOperator &I)
Instruction * visitUnreachableInst(UnreachableInst &I)
Instruction * foldOpIntoPhi(Instruction &I, PHINode *PN, bool AllowMultipleUses=false)
Given a binary operator, cast instruction, or select which has a PHI node as operand #0,...
void handleUnreachableFrom(Instruction *I, SmallVectorImpl< BasicBlock * > &Worklist)
Value * SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &PoisonElts, unsigned Depth=0, bool AllowMultipleUsers=false) override
The specified value produces a vector with any number of elements.
Instruction * visitFreeze(FreezeInst &I)
void handlePotentiallyDeadBlocks(SmallVectorImpl< BasicBlock * > &Worklist)
bool prepareWorklist(Function &F)
Perform early cleanup and prepare the InstCombine worklist.
Instruction * visitFree(CallInst &FI, Value *FreedOp)
Instruction * visitExtractValueInst(ExtractValueInst &EV)
void handlePotentiallyDeadSuccessors(BasicBlock *BB, BasicBlock *LiveSucc)
Instruction * visitUnconditionalBranchInst(BranchInst &BI)
Instruction * foldBinopWithRecurrence(BinaryOperator &BO)
Try to fold binary operators whose operands are simple interleaved recurrences to a single recurrence...
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Instruction * visitLandingPadInst(LandingPadInst &LI)
Instruction * visitReturnInst(ReturnInst &RI)
Instruction * visitSwitchInst(SwitchInst &SI)
Instruction * foldBinopWithPhiOperands(BinaryOperator &BO)
For a binary operator with 2 phi operands, try to hoist the binary operation before the phi.
bool mergeStoreIntoSuccessor(StoreInst &SI)
Try to transform: if () { *P = v1; } else { *P = v2 } or: *P = v1; if () { *P = v2; }...
Instruction * tryFoldInstWithCtpopWithNot(Instruction *I)
void CreateNonTerminatorUnreachable(Instruction *InsertAt)
Create and insert the idiom we use to indicate a block is unreachable without having to rewrite the C...
Value * pushFreezeToPreventPoisonFromPropagating(FreezeInst &FI)
bool run()
Run the combiner over the entire worklist until it is empty.
Instruction * foldVectorBinop(BinaryOperator &Inst)
Canonicalize the position of binops relative to shufflevector.
bool removeInstructionsBeforeUnreachable(Instruction &I)
Value * SimplifySelectsFeedingBinaryOp(BinaryOperator &I, Value *LHS, Value *RHS)
void tryToSinkInstructionDbgVariableRecords(Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock, BasicBlock *DestBlock, SmallVectorImpl< DbgVariableRecord * > &DPUsers)
void addDeadEdge(BasicBlock *From, BasicBlock *To, SmallVectorImpl< BasicBlock * > &Worklist)
Constant * unshuffleConstant(ArrayRef< int > ShMask, Constant *C, VectorType *NewCTy)
Find a constant NewC that has property: shuffle(NewC, ShMask) = C Returns nullptr if such a constant ...
Instruction * visitAllocSite(Instruction &FI)
Instruction * visitGetElementPtrInst(GetElementPtrInst &GEP)
Instruction * visitBranchInst(BranchInst &BI)
Value * tryFactorizationFolds(BinaryOperator &I)
This tries to simplify binary operations by factorizing out common terms (e.
Instruction * foldFreezeIntoRecurrence(FreezeInst &I, PHINode *PN)
Value * SimplifyDemandedUseFPClass(Value *V, FPClassTest DemandedMask, KnownFPClass &Known, Instruction *CxtI, unsigned Depth=0)
Attempts to replace V with a simpler value based on the demanded floating-point classes.
bool tryToSinkInstruction(Instruction *I, BasicBlock *DestBlock)
Try to move the specified instruction from its current block into the beginning of DestBlock,...
bool freezeOtherUses(FreezeInst &FI)
void freelyInvertAllUsersOf(Value *V, Value *IgnoredUser=nullptr)
Freely adapt every user of V as-if V was changed to !V.
The core instruction combiner logic.
SimplifyQuery SQ
const DataLayout & getDataLayout() const
IRBuilder< TargetFolder, IRBuilderCallbackInserter > BuilderTy
An IRBuilder that automatically inserts new instructions into the worklist.
bool isFreeToInvert(Value *V, bool WillInvertAllUses, bool &DoesConsume)
Return true if the specified value is free to invert (apply ~ to).
static unsigned getComplexity(Value *V)
Assign a complexity or rank value to LLVM Values.
TargetLibraryInfo & TLI
unsigned ComputeNumSignBits(const Value *Op, const Instruction *CxtI=nullptr, unsigned Depth=0) const
Instruction * InsertNewInstBefore(Instruction *New, BasicBlock::iterator Old)
Inserts an instruction New before instruction Old.
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
uint64_t MaxArraySizeForCombine
Maximum size of array considered when transforming.
static bool shouldAvoidAbsorbingNotIntoSelect(const SelectInst &SI)
void replaceUse(Use &U, Value *NewValue)
Replace use and add the previously used value to the worklist.
static bool isCanonicalPredicate(CmpPredicate Pred)
Predicate canonicalization reduces the number of patterns that need to be matched by other transforms...
InstructionWorklist & Worklist
A worklist of the instructions that need to be simplified.
Instruction * InsertNewInstWith(Instruction *New, BasicBlock::iterator Old)
Same as InsertNewInstBefore, but also sets the debug loc.
BranchProbabilityInfo * BPI
ReversePostOrderTraversal< BasicBlock * > & RPOT
const DataLayout & DL
DomConditionCache DC
const bool MinimizeSize
void computeKnownBits(const Value *V, KnownBits &Known, const Instruction *CxtI, unsigned Depth=0) const
std::optional< Instruction * > targetInstCombineIntrinsic(IntrinsicInst &II)
AssumptionCache & AC
void addToWorklist(Instruction *I)
Value * getFreelyInvertedImpl(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume, unsigned Depth)
Return nonnull value if V is free to invert under the condition of WillInvertAllUses.
SmallDenseSet< std::pair< const BasicBlock *, const BasicBlock * >, 8 > BackEdges
Backedges, used to avoid pushing instructions across backedges in cases where this may result in infi...
std::optional< Value * > targetSimplifyDemandedVectorEltsIntrinsic(IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3, std::function< void(Instruction *, unsigned, APInt, APInt &)> SimplifyAndSetOp)
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
DominatorTree & DT
static Constant * getSafeVectorConstantForBinop(BinaryOperator::BinaryOps Opcode, Constant *In, bool IsRHSConstant)
Some binary operators require special handling to avoid poison and undefined behavior.
SmallDenseSet< std::pair< BasicBlock *, BasicBlock * >, 8 > DeadEdges
Edges that are known to never be taken.
std::optional< Value * > targetSimplifyDemandedUseBitsIntrinsic(IntrinsicInst &II, APInt DemandedMask, KnownBits &Known, bool &KnownBitsComputed)
BuilderTy & Builder
bool isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const
Value * getFreelyInverted(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume)
bool isBackEdge(const BasicBlock *From, const BasicBlock *To)
void visit(Iterator Start, Iterator End)
Definition InstVisitor.h:87
The legacy pass manager's instcombine pass.
Definition InstCombine.h:68
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool runOnFunction(Function &F) override
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
InstructionWorklist - This is the worklist management logic for InstCombine and other simplification ...
void add(Instruction *I)
Add instruction to the worklist.
LLVM_ABI void dropUBImplyingAttrsAndMetadata(ArrayRef< unsigned > Keep={})
Drop any attributes or metadata that can cause immediate undefined behavior.
static bool isBitwiseLogicOp(unsigned Opcode)
Determine if the Opcode is and/or/xor.
LLVM_ABI void copyIRFlags(const Value *V, bool IncludeWrapFlags=true)
Convenience method to copy supported exact, fast-math, and (optionally) wrapping flags from V to this...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void setAAMetadata(const AAMDNodes &N)
Sets the AA metadata on this instruction from the AAMDNodes structure.
LLVM_ABI bool isAssociative() const LLVM_READONLY
Return true if the instruction is associative:
LLVM_ABI bool isCommutative() const LLVM_READONLY
Return true if the instruction is commutative:
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
LLVM_ABI void setFastMathFlags(FastMathFlags FMF)
Convenience function for setting multiple fast-math flags on this instruction, which must be an opera...
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
bool isTerminator() const
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
LLVM_ABI bool willReturn() const LLVM_READONLY
Return true if the instruction will return (unwinding is considered as a form of returning control fl...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
bool isBitwiseLogicOp() const
Return true if this is and/or/xor.
bool isShift() const
LLVM_ABI void dropPoisonGeneratingFlags()
Drops flags that may cause this instruction to evaluate to poison despite having non-poison inputs.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
bool isIntDivRem() const
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:319
A wrapper class for inspecting calls to intrinsic functions.
Invoke instruction.
static InvokeInst * Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value * > Args, const Twine &NameStr, InsertPosition InsertBefore=nullptr)
The landingpad instruction holds all of the information necessary to generate correct exception handl...
bool isCleanup() const
Return 'true' if this landingpad instruction is a cleanup.
unsigned getNumClauses() const
Get the number of clauses for this landing pad.
static LLVM_ABI LandingPadInst * Create(Type *RetTy, unsigned NumReservedClauses, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedClauses is a hint for the number of incoming clauses that this landingpad w...
LLVM_ABI void addClause(Constant *ClauseVal)
Add a catch or filter clause to the landing pad.
bool isCatch(unsigned Idx) const
Return 'true' if the clause and index Idx is a catch clause.
bool isFilter(unsigned Idx) const
Return 'true' if the clause and index Idx is a filter clause.
Constant * getClause(unsigned Idx) const
Get the value of the clause at index Idx.
void setCleanup(bool V)
Indicate that this landingpad instruction is a cleanup.
A function/module analysis which provides an empty LastRunTrackingInfo.
This is an alternative analysis pass to BlockFrequencyInfoWrapperPass.
static void getLazyBFIAnalysisUsage(AnalysisUsage &AU)
Helper for client passes to set up the analysis usage on behalf of this pass.
An instruction for reading from memory.
Value * getPointerOperand()
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Metadata node.
Definition Metadata.h:1078
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1442
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1448
Tracking metadata reference owned by Metadata.
Definition Metadata.h:900
This is the common base class for memset/memcpy/memmove.
static LLVM_ABI MemoryLocation getForDest(const MemIntrinsic *MI)
Return a location representing the destination of a memory set or transfer.
Root of the metadata hierarchy.
Definition Metadata.h:64
Value * getLHS() const
Value * getRHS() const
static ICmpInst::Predicate getPredicate(Intrinsic::ID ID)
Returns the comparison predicate underlying the intrinsic.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
MDNode * getScopeList() const
OptimizationRemarkEmitter legacy analysis pass.
The optimization diagnostic interface.
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition Operator.h:78
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property.
Definition Operator.h:111
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property.
Definition Operator.h:105
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
static LLVM_ABI PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
AnalysisType * getAnalysisIfAvailable() const
getAnalysisIfAvailable<AnalysisType>() - Subclasses use this function to get analysis information tha...
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
Definition Constants.h:1468
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
Definition Analysis.h:151
PreservedAnalyses & preserve()
Mark an analysis as preserved.
Definition Analysis.h:132
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
bool hasProfileSummary() const
Returns true if profile summary is available.
A global registry used in conjunction with static constructors to make pluggable components (like tar...
Definition Registry.h:44
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, InsertPosition InsertBefore=nullptr)
This class represents the LLVM 'select' instruction.
const Value * getFalseValue() const
const Value * getCondition() const
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, const Instruction *MDFrom=nullptr)
const Value * getTrueValue() const
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:150
This instruction constructs a fixed permutation of two input vectors.
size_type size() const
Definition SmallPtrSet.h:99
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
Definition SetVector.h:338
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition SmallSet.h:133
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:183
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
typename SuperClass::iterator iterator
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
Multiway switch.
TargetFolder - Create constants with target dependent folding.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
bool has(LibFunc F) const
Tests whether a library function is available.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
Wrapper pass for TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:273
LLVM_ABI bool isScalableTy(SmallPtrSetImpl< const Type * > &Visited) const
Return true if this is a type whose size is a known multiple of vscale.
Definition Type.cpp:62
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:267
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:295
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:352
bool isStructTy() const
True if this is an instance of StructType.
Definition Type.h:261
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:198
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition Type.h:311
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:231
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:294
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:240
LLVM_ABI const fltSemantics & getFltSemantics() const
Definition Type.cpp:107
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
Use * op_iterator
Definition User.h:279
op_range operands()
Definition User.h:292
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition User.cpp:21
op_iterator op_begin()
Definition User.h:284
const Use & getOperandUse(unsigned i) const
Definition User.h:245
Value * getOperand(unsigned i) const
Definition User.h:232
unsigned getNumOperands() const
Definition User.h:254
op_iterator op_end()
Definition User.h:286
LLVM_ABI bool isDroppable() const
A droppable user is a user for which uses can be dropped without affecting correctness and should be ...
Definition User.cpp:115
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
const Value * stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, APInt &Offset) const
This is a wrapper around stripAndAccumulateConstantOffsets with the in-bounds requirement set to fals...
Definition Value.h:759
LLVM_ABI bool hasOneUser() const
Return true if there is exactly one user of this value.
Definition Value.cpp:166
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:439
iterator_range< user_iterator > users()
Definition Value.h:426
bool hasUseList() const
Check if this Value has a use-list.
Definition Value.h:344
LLVM_ABI bool hasNUses(unsigned N) const
Return true if this Value has exactly N uses.
Definition Value.cpp:150
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:701
bool use_empty() const
Definition Value.h:346
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.cpp:1099
LLVM_ABI uint64_t getPointerDereferenceableBytes(const DataLayout &DL, bool &CanBeNull, bool &CanBeFreed) const
Returns the number of bytes known to be dereferenceable for the pointer value.
Definition Value.cpp:881
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:396
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Value handle that is nullable, but tries to track the Value.
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition ilist_node.h:34
reverse_self_iterator getReverseIterator()
Definition ilist_node.h:126
self_iterator getIterator()
Definition ilist_node.h:123
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
A raw_ostream that writes to an std::string.
Changed
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Abstract Attribute helper functions.
Definition Attributor.h:165
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
cst_pred_ty< is_all_ones > m_AllOnes()
Match an integer or vector with all bits set.
class_match< PoisonValue > m_Poison()
Match an arbitrary poison constant.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
PtrAdd_match< PointerOpTy, OffsetOpTy > m_PtrAdd(const PointerOpTy &PointerOp, const OffsetOpTy &OffsetOp)
Matches GEP with i8 source element type.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
CmpClass_match< LHS, RHS, FCmpInst > m_FCmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
OneOps_match< OpTy, Instruction::Freeze > m_Freeze(const OpTy &Op)
Matches FreezeInst.
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
br_match m_UnconditionalBr(BasicBlock *&Succ)
ap_match< APInt > m_APIntAllowPoison(const APInt *&Res)
Match APInt while allowing poison in splat vector constants.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
BinOpPred_match< LHS, RHS, is_idiv_op > m_IDiv(const LHS &L, const RHS &R)
Matches integer division operations.
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
DisjointOr_match< LHS, RHS > m_DisjointOr(const LHS &L, const RHS &R)
constantexpr_match m_ConstantExpr()
Match a constant expression or a constant that contains a constant expression.
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
ap_match< APFloat > m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
cst_pred_ty< is_nonnegative > m_NonNegative()
Match an integer or vector of non-negative values.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
IntrinsicID_match m_Intrinsic()
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
ExtractValue_match< Ind, Val_t > m_ExtractValue(const Val_t &V)
Match a single index ExtractValue instruction.
match_combine_and< LTy, RTy > m_CombineAnd(const LTy &L, const RTy &R)
Combine two pattern matchers matching L && R.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
NNegZExt_match< OpTy > m_NNegZExt(const OpTy &Op)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
ThreeOps_match< decltype(m_Value()), LHS, RHS, Instruction::Select, true > m_c_Select(const LHS &L, const RHS &R)
Match Select(C, LHS, RHS) or Select(C, RHS, LHS)
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
BinaryOp_match< LHS, RHS, Instruction::UDiv > m_UDiv(const LHS &L, const RHS &R)
brc_match< Cond_t, bind_ty< BasicBlock >, bind_ty< BasicBlock > > m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F)
match_immconstant_ty m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
match_combine_or< BinaryOp_match< LHS, RHS, Instruction::Add >, DisjointOr_match< LHS, RHS > > m_AddLike(const LHS &L, const RHS &R)
Match either "add" or "or disjoint".
CastInst_match< OpTy, UIToFPInst > m_UIToFP(const OpTy &Op)
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
match_combine_or< CastInst_match< OpTy, SExtInst >, NNegZExt_match< OpTy > > m_SExtLike(const OpTy &Op)
Match either "sext" or "zext nneg".
BinaryOp_match< LHS, RHS, Instruction::SDiv > m_SDiv(const LHS &L, const RHS &R)
match_combine_or< OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoSignedWrap >, DisjointOr_match< LHS, RHS > > m_NSWAddLike(const LHS &L, const RHS &R)
Match either "add nsw" or "or disjoint".
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
AnyBinaryOp_match< LHS, RHS, true > m_c_BinOp(const LHS &L, const RHS &R)
Matches a BinaryOperator with LHS and RHS in either order.
CastInst_match< OpTy, SIToFPInst > m_SIToFP(const OpTy &Op)
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
BinOpPred_match< LHS, RHS, is_shift_op > m_Shift(const LHS &L, const RHS &R)
Matches shift operations.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
cstfp_pred_ty< is_non_zero_fp > m_NonZeroFP()
Match a floating-point non-zero.
m_Intrinsic_Ty< Opnd0 >::Ty m_VecReverse(const Opnd0 &Op0)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
match_combine_or< match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > >, match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > > > m_MaxOrMin(const LHS &L, const RHS &R)
auto m_Undef()
Match an arbitrary undef constant.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
match_combine_or< OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap >, DisjointOr_match< LHS, RHS > > m_NUWAddLike(const LHS &L, const RHS &R)
Match either "add nuw" or "or disjoint".
CastOperator_match< OpTy, Instruction::PtrToInt > m_PtrToInt(const OpTy &Op)
Matches PtrToInt.
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
initializer< Ty > init(const Ty &Val)
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
LLVM_ABI Intrinsic::ID getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID)
@ Offset
Definition DWP.cpp:477
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition STLExtras.h:829
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
void stable_sort(R &&Range)
Definition STLExtras.h:2058
LLVM_ABI void initializeInstructionCombiningPassPass(PassRegistry &)
LLVM_ABI unsigned removeAllNonTerminatorAndEHPadInstructions(BasicBlock *BB)
Remove all instructions from a basic block other than its terminator and any present EH pad instructi...
Definition Local.cpp:2485
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1725
LLVM_ABI Value * simplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef< Value * > Indices, GEPNoWrapFlags NW, const SimplifyQuery &Q)
Given operands for a GetElementPtrInst, fold the result or return null.
LLVM_ABI Constant * getInitialValueOfAllocation(const Value *V, const TargetLibraryInfo *TLI, Type *Ty)
If this is a call to an allocation function that initializes memory to a fixed value,...
bool succ_empty(const Instruction *I)
Definition CFG.h:257
LLVM_ABI Value * simplifyFreezeInst(Value *Op, const SimplifyQuery &Q)
Given an operand for a Freeze, see if we can fold the result.
LLVM_ABI FunctionPass * createInstructionCombiningPass()
LLVM_ABI void findDbgValues(Value *V, SmallVectorImpl< DbgVariableRecord * > &DbgVariableRecords)
Finds the dbg.values describing a value.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2472
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition Utils.cpp:1724
auto successors(const MachineBasicBlock *BB)
LLVM_ABI Constant * ConstantFoldInstruction(const Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldInstruction - Try to constant fold the specified instruction.
LLVM_ABI bool isRemovableAlloc(const CallBase *V, const TargetLibraryInfo *TLI)
Return true if this is a call to an allocation function that does not have side effects that we are r...
LLVM_ABI std::optional< StringRef > getAllocationFamily(const Value *I, const TargetLibraryInfo *TLI)
If a function is part of an allocation family (e.g.
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
LLVM_ABI Value * lowerObjectSizeCall(IntrinsicInst *ObjectSize, const DataLayout &DL, const TargetLibraryInfo *TLI, bool MustSucceed)
Try to turn a call to @llvm.objectsize into an integer value of the given Type.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
LLVM_ABI Value * simplifyInstructionWithOperands(Instruction *I, ArrayRef< Value * > NewOps, const SimplifyQuery &Q)
Like simplifyInstruction but the operands of I are replaced with NewOps.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2136
LLVM_ABI Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:632
gep_type_iterator gep_type_end(const User *GEP)
LLVM_ABI Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
LLVM_ABI Value * getReallocatedOperand(const CallBase *CB)
If this is a call to a realloc function, return the reallocated operand.
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
Definition APFloat.h:1537
LLVM_ABI bool isAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI)
Tests if a value is a call or invoke to a library function that allocates memory (either malloc,...
LLVM_ABI bool handleUnreachableTerminator(Instruction *I, SmallVectorImpl< Value * > &PoisonedValues)
If a terminator in an unreachable basic block has an operand of type Instruction, transform it into p...
Definition Local.cpp:2468
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:202
LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start, Value *&Step)
Attempt to match a simple first order recurrence cycle of the form: iv = phi Ty [Start,...
LLVM_ABI Value * simplifyAddInst(Value *LHS, Value *RHS, bool IsNSW, bool IsNUW, const SimplifyQuery &Q)
Given operands for an Add, fold the result or return null.
LLVM_ABI Constant * ConstantFoldConstant(const Constant *C, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldConstant - Fold the constant using the specified DataLayout.
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
constexpr bool has_single_bit(T Value) noexcept
Definition bit.h:147
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1732
LLVM_ABI bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
Definition Local.cpp:402
LLVM_ABI bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
LLVM_ABI Value * emitGEPOffset(IRBuilderBase *Builder, const DataLayout &DL, User *GEP, bool NoAssumptions=false)
Given a getelementptr instruction/constantexpr, emit the code necessary to compute the offset from th...
Definition Local.cpp:22
constexpr unsigned MaxAnalysisRecursionDepth
auto reverse(ContainerTy &&C)
Definition STLExtras.h:406
bool isModSet(const ModRefInfo MRI)
Definition ModRef.h:49
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI bool LowerDbgDeclare(Function &F)
Lowers dbg.declare records into appropriate set of dbg.value records.
Definition Local.cpp:1795
LLVM_ABI bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
generic_gep_type_iterator<> gep_type_iterator
LLVM_ABI void ConvertDebugDeclareToDebugValue(DbgVariableRecord *DVR, StoreInst *SI, DIBuilder &Builder)
Inserts a dbg.value record before a store to an alloca'd value that has an associated dbg....
Definition Local.cpp:1662
LLVM_ABI void salvageDebugInfoForDbgValues(Instruction &I, ArrayRef< DbgVariableRecord * > DPInsns)
Implementation of salvageDebugInfo, applying only to instructions in Insns, rather than all debug use...
Definition Local.cpp:2037
LLVM_ABI Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
LLVM_ABI bool canCreateUndefOrPoison(const Operator *Op, bool ConsiderFlagsAndMetadata=true)
canCreateUndefOrPoison returns true if Op can create undef or poison from non-undef & non-poison oper...
LLVM_ABI EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI Value * simplifyExtractValueInst(Value *Agg, ArrayRef< unsigned > Idxs, const SimplifyQuery &Q)
Given operands for an ExtractValueInst, fold the result or return null.
LLVM_ABI Constant * ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL)
Attempt to constant fold a binary operation with the specified operands.
LLVM_ABI bool replaceAllDbgUsesWith(Instruction &From, Value &To, Instruction &DomPoint, DominatorTree &DT)
Point debug users of From to To or salvage them.
Definition Local.cpp:2414
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
constexpr int PoisonMaskElem
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:323
ModRefInfo
Flags indicating whether a memory access modifies or references memory.
Definition ModRef.h:28
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
@ ModRef
The access may reference and may modify the value stored in memory.
Definition ModRef.h:36
@ Mod
The access may modify the value stored in memory.
Definition ModRef.h:34
@ NoModRef
The access neither references nor modifies the value stored in memory.
Definition ModRef.h:30
TargetTransformInfo TTI
LLVM_ABI Value * simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a BinaryOperator, fold the result or return null.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
DWARFExpression::Operation Op
bool isSafeToSpeculativelyExecuteWithVariableReplaced(const Instruction *I, bool IgnoreUBImplyingAttrs=true)
Don't use information from its non-constant operands.
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI Value * getFreedOperand(const CallBase *CB, const TargetLibraryInfo *TLI)
If this if a call to a free function, return the freed operand.
constexpr unsigned BitWidth
LLVM_ABI bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
LLVM_ABI Constant * getLosslessInvCast(Constant *C, Type *InvCastTo, unsigned CastOp, const DataLayout &DL, PreservedCastFlags *Flags=nullptr)
Try to cast C to InvC losslessly, satisfying CastOp(InvC) equals C, or CastOp(InvC) is a refined valu...
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition STLExtras.h:1961
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
gep_type_iterator gep_type_begin(const User *GEP)
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1897
cl::opt< bool > ProfcheckDisableMetadataFixes("profcheck-disable-metadata-fixes", cl::Hidden, cl::init(false), cl::desc("Disable metadata propagation fixes discovered through Issue #147390"))
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
bool equal(L &&LRange, R &&RRange)
Wrapper function around std::equal to detect if pair-wise elements between two ranges are the same.
Definition STLExtras.h:2088
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
AAResults AliasAnalysis
Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
LLVM_ABI void initializeInstCombine(PassRegistry &)
Initialize all passes linked into the InstCombine library.
LLVM_ABI void findDbgUsers(Value *V, SmallVectorImpl< DbgVariableRecord * > &DbgVariableRecords)
Finds the debug info records describing a value.
LLVM_ABI Constant * ConstantFoldBinaryInstruction(unsigned Opcode, Constant *V1, Constant *V2)
bool isRefSet(const ModRefInfo MRI)
Definition ModRef.h:52
LLVM_ABI std::optional< bool > isImpliedCondition(const Value *LHS, const Value *RHS, const DataLayout &DL, bool LHSIsTrue=true, unsigned Depth=0)
Return true if RHS is known to be implied true by LHS.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:180
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:869
#define N
unsigned countMinLeadingOnes() const
Returns the minimum number of leading one bits.
Definition KnownBits.h:251
unsigned getBitWidth() const
Get the bit width of this value.
Definition KnownBits.h:44
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:248
A CRTP mix-in to automatically provide informational APIs needed for passes.
Definition PassManager.h:70
SimplifyQuery getWithInstruction(const Instruction *I) const