42#define LV_NAME "loop-vectorize"
43#define DEBUG_TYPE LV_NAME
47 case VPWidenMemoryInstructionSC: {
48 return cast<VPWidenMemoryInstructionRecipe>(
this)->isStore();
54 case VPBranchOnMaskSC:
55 case VPScalarIVStepsSC:
60 case VPWidenCanonicalIVSC:
63 case VPWidenIntOrFpInductionSC:
66 case VPWidenSelectSC: {
70 assert((!
I || !
I->mayWriteToMemory()) &&
71 "underlying instruction may write to memory");
81 case VPWidenMemoryInstructionSC: {
82 return !cast<VPWidenMemoryInstructionRecipe>(
this)->isStore();
87 ->mayReadFromMemory();
88 case VPBranchOnMaskSC:
89 case VPScalarIVStepsSC:
94 case VPWidenCanonicalIVSC:
97 case VPWidenIntOrFpInductionSC:
100 case VPWidenSelectSC: {
104 assert((!
I || !
I->mayReadFromMemory()) &&
105 "underlying instruction may read from memory");
116 case VPPredInstPHISC:
118 case VPInstructionSC:
119 switch (cast<VPInstruction>(
this)->
getOpcode()) {
120 case Instruction::ICmp:
131 ->mayHaveSideEffects();
134 case VPScalarIVStepsSC:
135 case VPWidenCanonicalIVSC:
138 case VPWidenIntOrFpInductionSC:
140 case VPWidenPointerInductionSC:
142 case VPWidenSelectSC: {
146 assert((!
I || !
I->mayHaveSideEffects()) &&
147 "underlying instruction has side-effects");
150 case VPWidenMemoryInstructionSC:
151 assert(cast<VPWidenMemoryInstructionRecipe>(
this)
154 "mayHaveSideffects result for ingredient differs from this "
157 case VPReplicateSC: {
158 auto *R = cast<VPReplicateRecipe>(
this);
159 return R->getUnderlyingInstr()->mayHaveSideEffects();
174 "the middle block must not have any successors");
180#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
191 assert(!Parent &&
"Recipe already in some VPBasicBlock");
193 "Insertion position not in any VPBasicBlock");
200 assert(!Parent &&
"Recipe already in some VPBasicBlock");
207 assert(!Parent &&
"Recipe already in some VPBasicBlock");
209 "Insertion position not in any VPBasicBlock");
237 assert(OpType == OperationType::FPMathOp &&
238 "recipe doesn't have fast math flags");
257 "only ICmp predicates supported at the moment");
261 std::initializer_list<VPValue *>
Operands,
266 assert(isFPMathOp() &&
"this op can't take fast-math flags");
285 case Instruction::ICmp: {
290 case Instruction::Select: {
306 {VIVElem0, ScalarTC},
nullptr, Name);
348 return State.
get(
this, 0);
413bool VPInstruction::isFPMathOp()
const {
416 return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
417 Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
418 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
419 Opcode == Instruction::FCmp || Opcode == Instruction::Select;
428 "Recipe not a FPMathOp but has fast-math flags?");
431 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
432 Value *GeneratedValue = generateInstruction(State, Part);
435 assert(GeneratedValue &&
"generateInstruction must produce a value");
436 State.
set(
this, GeneratedValue, Part);
440#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
448 O << Indent <<
"EMIT ";
460 O <<
"combined load";
463 O <<
"combined store";
466 O <<
"active lane mask";
469 O <<
"first-order splice";
475 O <<
"branch-on-cond";
478 O <<
"TC > VF ? TC - VF : 0";
484 O <<
"branch-on-count";
503 assert(!isa<DbgInfoIntrinsic>(CI) &&
504 "DbgInfoIntrinsic should have been dropped during VPlan construction");
510 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
529 Arg = State.
get(
I.value(), Part);
540 assert(VectorF &&
"Can't retrieve vector intrinsic.");
543 assert(Variant !=
nullptr &&
"Can't create vector function.");
549 CI.getOperandBundlesAsDefs(OpBundles);
552 if (isa<FPMathOperator>(V))
553 V->copyFastMathFlags(&CI);
555 State.
set(
this, V, Part);
560#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
563 O << Indent <<
"WIDEN-CALL ";
566 if (CI->getType()->isVoidTy())
573 O <<
"call @" << CI->getCalledFunction()->getName() <<
"(";
577 if (VectorIntrinsicID)
578 O <<
" (using vector intrinsic)";
580 O <<
" (using library function";
582 O <<
": " << Variant->
getName();
589 O << Indent <<
"WIDEN-SELECT ";
611 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
616 State.
set(
this, Sel, Part);
621VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy(
632#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
635 case OperationType::Cmp:
638 case OperationType::PossiblyExactOp:
642 case OperationType::OverflowingBinOp:
648 case OperationType::FPMathOp:
651 case OperationType::GEPOp:
655 case OperationType::Other:
667 case Instruction::Call:
668 case Instruction::Br:
669 case Instruction::PHI:
670 case Instruction::GetElementPtr:
671 case Instruction::Select:
673 case Instruction::UDiv:
674 case Instruction::SDiv:
675 case Instruction::SRem:
676 case Instruction::URem:
677 case Instruction::Add:
678 case Instruction::FAdd:
679 case Instruction::Sub:
680 case Instruction::FSub:
681 case Instruction::FNeg:
682 case Instruction::Mul:
683 case Instruction::FMul:
684 case Instruction::FDiv:
685 case Instruction::FRem:
686 case Instruction::Shl:
687 case Instruction::LShr:
688 case Instruction::AShr:
689 case Instruction::And:
690 case Instruction::Or:
691 case Instruction::Xor: {
693 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
700 if (
auto *VecOp = dyn_cast<Instruction>(V))
704 State.
set(
this, V, Part);
710 case Instruction::Freeze: {
711 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
715 State.
set(
this, Freeze, Part);
719 case Instruction::ICmp:
720 case Instruction::FCmp: {
722 bool FCmp = Opcode == Instruction::FCmp;
723 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
736 State.
set(
this,
C, Part);
752 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
755 "inferred type and type from generated instructions do not match");
760#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
763 O << Indent <<
"WIDEN ";
778 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
779 if (Part > 0 &&
Op->isLiveIn()) {
781 State.
set(
this, State.
get(
this, 0), Part);
786 State.
set(
this, Cast, Part);
791#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
794 O << Indent <<
"WIDEN-CAST ";
812 auto *ValVTy = cast<VectorType>(Val->
getType());
817 "Induction Step must be an integer or FP");
825 Type *InitVecValSTy =
835 InitVec = Builder.
CreateAdd(InitVec, StartIdxSplat);
841 return Builder.
CreateAdd(Val, Step,
"induction");
845 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
846 "Binary Opcode should be specified for FP induction");
848 InitVec = Builder.
CreateFAdd(InitVec, StartIdxSplat);
852 return Builder.
CreateBinOp(BinOp, Val, MulOp,
"induction");
867 return B.CreateUIToFP(RuntimeVF, FTy);
871 assert(!State.
Instance &&
"Int or FP induction being replicated.");
877 assert(IV->
getType() ==
ID.getStartValue()->getType() &&
"Types must match");
882 Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
886 if (
ID.getInductionBinOp() && isa<FPMathOperator>(
ID.getInductionBinOp()))
892 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
893 "Expected either an induction phi-node or a truncate of it!");
896 auto CurrIP = Builder.
saveIP();
899 if (isa<TruncInst>(EntryVal)) {
900 assert(Start->getType()->isIntegerTy() &&
901 "Truncation requires an integer type");
902 auto *TruncType = cast<IntegerType>(EntryVal->
getType());
904 Start = Builder.
CreateCast(Instruction::Trunc, Start, TruncType);
910 SplatStart, Zero, Step,
ID.getInductionOpcode(), State.
VF, State.
Builder);
917 AddOp = Instruction::Add;
918 MulOp = Instruction::Mul;
920 AddOp =
ID.getInductionOpcode();
921 MulOp = Instruction::FMul;
950 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
951 State.
set(
this, LastInduction, Part);
953 if (isa<TruncInst>(EntryVal))
956 LastInduction = cast<Instruction>(
957 Builder.
CreateBinOp(AddOp, LastInduction, SplatVF,
"step.add"));
961 LastInduction->
setName(
"vec.ind.next");
971#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
974 O << Indent <<
"WIDEN-INDUCTION";
978 O <<
" +\n" << Indent <<
"\" ";
996 return StartC && StartC->isZero() && StepC && StepC->isOne();
999#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1004 O << Indent <<
"= DERIVED-IV ";
1012 O <<
" (truncated to " << *TruncResultTy <<
")";
1031 if (BaseIVTy != Step->
getType()) {
1035 "Truncation requires an integer step");
1044 AddOp = Instruction::Add;
1045 MulOp = Instruction::Mul;
1047 AddOp = InductionOpcode;
1048 MulOp = Instruction::FMul;
1057 Type *VecIVTy =
nullptr;
1058 Value *UnitStepVec =
nullptr, *SplatStep =
nullptr, *SplatIV =
nullptr;
1067 unsigned StartPart = 0;
1068 unsigned EndPart = State.
UF;
1069 unsigned StartLane = 0;
1073 EndPart = StartPart + 1;
1074 StartLane = State.
Instance->Lane.getKnownLane();
1075 EndLane = StartLane + 1;
1077 for (
unsigned Part = StartPart; Part < EndPart; ++Part) {
1082 auto *InitVec = Builder.
CreateAdd(SplatStartIdx, UnitStepVec);
1087 State.
set(
this,
Add, Part);
1096 for (
unsigned Lane = StartLane; Lane < EndLane; ++Lane) {
1102 "Expected StartIdx to be folded to a constant when VF is not "
1111#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1116 O <<
" = SCALAR-STEPS ";
1130 if (areAllOperandsInvariant()) {
1150 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1152 State.
set(
this, EntryPart, Part);
1163 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1166 auto *
Ptr = isPointerLoopInvariant()
1175 if (isIndexLoopInvariant(
I - 1))
1186 "NewGEP is not a pointer vector");
1187 State.
set(
this, NewGEP, Part);
1193#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1196 O << Indent <<
"WIDEN-GEP ";
1197 O << (isPointerLoopInvariant() ?
"Inv" :
"Var");
1199 O <<
"[" << (isIndexLoopInvariant(
I) ?
"Inv" :
"Var") <<
"]";
1203 O <<
" = getelementptr";
1228 for (
unsigned In = 0; In < NumIncoming; ++In) {
1229 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1244 for (
unsigned Part = 0; Part < State.
UF; ++Part)
1245 State.
set(
this, Entry[Part], Part);
1248#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1251 O << Indent <<
"BLEND ";
1271 O << Indent <<
"REDUCE ";
1286 O <<
" (with final reduction value stored in invariant address sank "
1295 if (
auto *PredR = dyn_cast<VPPredInstPHIRecipe>(U))
1296 return any_of(PredR->users(), [PredR](
const VPUser *U) {
1297 return !U->usesScalars(PredR);
1303#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1306 O << Indent << (IsUniform ?
"CLONE " :
"REPLICATE ");
1315 O <<
"@" << CB->getCalledFunction()->getName() <<
"(";
1333 assert(State.
Instance &&
"Branch on Mask works only on single instance.");
1335 unsigned Part = State.
Instance->Part;
1336 unsigned Lane = State.
Instance->Lane.getKnownLane();
1338 Value *ConditionBit =
nullptr;
1341 ConditionBit = State.
get(BlockInMask, Part);
1351 assert(isa<UnreachableInst>(CurrentTerminator) &&
1352 "Expected to replace unreachable terminator with conditional branch.");
1359 assert(State.
Instance &&
"Predicated instruction PHI works per instance.");
1364 assert(PredicatingBB &&
"Predicated block has no single predecessor.");
1366 "operand must be VPReplicateRecipe");
1374 unsigned Part = State.
Instance->Part;
1382 State.
reset(
this, VPhi, Part);
1384 State.
set(
this, VPhi, Part);
1393 Phi->addIncoming(ScalarPredInst, PredicatedBB);
1404#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1407 O << Indent <<
"PHI-PREDICATED-INSTRUCTION ";
1415 O << Indent <<
"WIDEN ";
1435 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1436 State.
set(
this, EntryPart, Part);
1439#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1442 O << Indent <<
"EMIT ";
1444 O <<
" = CANONICAL-INDUCTION ";
1464 return StepC && StepC->
isOne();
1468 return IsScalarAfterVectorization &&
1472#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1475 O << Indent <<
"EMIT ";
1477 O <<
" = WIDEN-POINTER-INDUCTION ";
1479 O <<
", " << *IndDesc.
getStep();
1491 "Same SCEV expanded multiple times");
1493 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1494 State.
set(
this, Res, {Part, 0});
1497#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1500 O << Indent <<
"EMIT ";
1502 O <<
" = EXPAND SCEV " << *Expr;
1514 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
1521 Value *CanonicalVectorIV = Builder.
CreateAdd(VStart, VStep,
"vec.iv");
1522 State.
set(
this, CanonicalVectorIV, Part);
1526#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1529 O << Indent <<
"EMIT ";
1531 O <<
" = WIDEN-CANONICAL-INDUCTION ";
1537 auto &Builder = State.
Builder;
1542 ? VectorInit->getType()
1552 auto *LastIdx = Builder.
CreateSub(RuntimeVF, One);
1561 State.
set(
this, EntryPart, 0);
1564#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1567 O << Indent <<
"FIRST-ORDER-RECURRENCE-PHI ";
1576 auto &Builder = State.
Builder;
1582 bool ScalarPHI = State.
VF.
isScalar() || IsInLoop;
1588 "recipe must be in the vector loop header");
1589 unsigned LastPartForNewPhi =
isOrdered() ? 1 : State.
UF;
1590 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1593 State.
set(
this, EntryPart, Part);
1603 Value *Iden =
nullptr;
1629 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1630 Value *EntryPart = State.
get(
this, Part);
1633 Value *StartVal = (Part == 0) ? StartV : Iden;
1634 cast<PHINode>(EntryPart)->addIncoming(StartVal, VectorPH);
1638#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1641 O << Indent <<
"WIDEN-REDUCTION-PHI ";
1651 "Non-native vplans are not expected to have VPWidenPHIRecipes.");
1656 State.
set(
this, VecPhi, 0);
1659#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1662 O << Indent <<
"WIDEN-PHI ";
1684 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
1690 State.
set(
this, EntryPart, Part);
1694#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1697 O << Indent <<
"ACTIVE-LANE-MASK-PHI ";
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
mir Rename Register Operands
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static Value * getStepVector(Value *Val, Value *StartIdx, Value *Step, Instruction::BinaryOps BinOp, ElementCount VF, IRBuilderBase &Builder)
This function adds (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step,...
static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)
A helper function that returns an integer or floating-point constant with value C.
static Value * getRuntimeVFAsFloat(IRBuilderBase &B, Type *FTy, ElementCount VF)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
This file contains the declarations of the Vectorization Plan base classes:
static constexpr uint32_t Opcode
static const uint32_t IV[8]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
static StringRef getPredicateName(Predicate P)
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
This is the shared class of boolean and integer constants.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
constexpr bool isVector() const
One or more elements.
constexpr bool isScalar() const
Exactly one element.
Convenience struct for specifying and reasoning about fast-math flags.
void setAllowContract(bool B=true)
bool noSignedZeros() const
void setAllowReciprocal(bool B=true)
bool allowReciprocal() const
void print(raw_ostream &O) const
Print fast-math flags to O.
void setNoSignedZeros(bool B=true)
bool allowReassoc() const
Flag queries.
void setNoNaNs(bool B=true)
void setAllowReassoc(bool B=true)
Flag setters.
void setApproxFunc(bool B=true)
void setNoInfs(bool B=true)
bool allowContract() const
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Common base class shared among various IRBuilders.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")
Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
ConstantInt * getTrue()
Get the constant value for i1 true.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateFreeze(Value *V, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
BasicBlock * GetInsertBlock() const
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="")
InsertPoint saveIP() const
Returns the current insert point.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Value * CreateNAryOp(unsigned Opc, ArrayRef< Value * > Ops, const Twine &Name="", MDNode *FPMathTag=nullptr)
Create either a UnaryOperator or BinaryOperator depending on Opc.
LLVMContext & getContext() const
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
void restoreIP(InsertPoint IP)
Sets the current insert point to a previously-saved location.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_IntInduction
Integer induction variable. Step = C.
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
BlockT * getHeader() const
void print(raw_ostream &OS, const SlotIndexes *=nullptr, bool IsStandalone=true) const
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
FastMathFlags getFastMathFlags() const
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
RecurKind getRecurrenceKind() const
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const
Returns identity corresponding to the RecurrenceKind.
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
This class uses information about analyze scalars to rewrite expressions in canonical form.
Type * getType() const
Return the LLVM type of this SCEV expression.
This class provides computation of slot numbers for LLVM Assembly writing.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Value * getOperand(unsigned i) const
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPRegionBlock * getParent()
size_t getNumSuccessors() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
VPValue * getMask() const
Return the mask used by this recipe.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step, Type *Ty) const
Check if the induction described by Kind, /p Start and Step is canonical, i.e.
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
Type * getScalarType() const
Returns the scalar type of the induction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
unsigned getVPDefID() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
VPValue * getCanonicalIV() const
VPValue * getStartValue() const
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
unsigned getOpcode() const
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
void execute(VPTransformState &State) override
Generate the instruction.
static VPLane getLastLaneForVF(const ElementCount &VF)
static VPLane getFirstLane()
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the VPLiveOut to O.
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Class to record LLVM IR flag for a recipe along with it.
void setFlags(Instruction *I) const
Set the IR flags for I.
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
bool hasNoUnsignedWrap() const
void printFlags(raw_ostream &O) const
CmpInst::Predicate getPredicate() const
bool hasNoSignedWrap() const
FastMathFlags getFastMathFlags() const
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCondOp() const
The VPValue of the condition for the block.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
unsigned getNumOperands() const
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
friend class VPInstruction
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
void execute(VPTransformState &State) override
Produce widened copies of the cast.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
VPValue * getStepValue()
Returns the step value of the induction.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isStore() const
Returns true if this recipe is a store.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool onlyScalarsGenerated(ElementCount VF)
Returns true if only scalar values will be generated.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
StringRef getName() const
Return a constant reference to the value's name.
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
base_list_type::iterator iterator
iterator erase(iterator where)
pointer remove(iterator &IT)
iterator insertAfter(iterator where, pointer New)
iterator insert(iterator where, pointer New)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
NodeAddr< PhiNode * > Phi
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
void execute(VPTransformState &State) override
Generate the phi nodes.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
bool isInvariantCond() const
VPValue * getCond() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.