43#define LV_NAME "loop-vectorize"
44#define DEBUG_TYPE LV_NAME
49 return cast<VPInterleaveRecipe>(
this)->getNumStoreOperands() > 0;
50 case VPWidenMemoryInstructionSC: {
51 return cast<VPWidenMemoryInstructionRecipe>(
this)->isStore();
57 case VPBranchOnMaskSC:
58 case VPScalarIVStepsSC:
63 case VPWidenCanonicalIVSC:
66 case VPWidenIntOrFpInductionSC:
69 case VPWidenSelectSC: {
73 assert((!
I || !
I->mayWriteToMemory()) &&
74 "underlying instruction may write to memory");
84 case VPWidenMemoryInstructionSC: {
85 return !cast<VPWidenMemoryInstructionRecipe>(
this)->isStore();
90 ->mayReadFromMemory();
91 case VPBranchOnMaskSC:
92 case VPScalarIVStepsSC:
97 case VPWidenCanonicalIVSC:
100 case VPWidenIntOrFpInductionSC:
103 case VPWidenSelectSC: {
107 assert((!
I || !
I->mayReadFromMemory()) &&
108 "underlying instruction may read from memory");
119 case VPPredInstPHISC:
122 case VPInstructionSC:
123 switch (cast<VPInstruction>(
this)->
getOpcode()) {
124 case Instruction::Or:
125 case Instruction::ICmp:
126 case Instruction::Select:
136 ->mayHaveSideEffects();
139 case VPScalarIVStepsSC:
140 case VPWidenCanonicalIVSC:
143 case VPWidenIntOrFpInductionSC:
145 case VPWidenPointerInductionSC:
147 case VPWidenSelectSC: {
151 assert((!
I || !
I->mayHaveSideEffects()) &&
152 "underlying instruction has side-effects");
157 case VPWidenMemoryInstructionSC:
158 assert(cast<VPWidenMemoryInstructionRecipe>(
this)
161 "mayHaveSideffects result for ingredient differs from this "
164 case VPReplicateSC: {
165 auto *R = cast<VPReplicateRecipe>(
this);
166 return R->getUnderlyingInstr()->mayHaveSideEffects();
181 "the middle block must not have any successors");
187#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
198 assert(!Parent &&
"Recipe already in some VPBasicBlock");
200 "Insertion position not in any VPBasicBlock");
206 assert(!Parent &&
"Recipe already in some VPBasicBlock");
212 assert(!Parent &&
"Recipe already in some VPBasicBlock");
214 "Insertion position not in any VPBasicBlock");
241 assert(OpType == OperationType::FPMathOp &&
242 "recipe doesn't have fast math flags");
260 assert(Opcode == Instruction::ICmp &&
261 "only ICmp predicates supported at the moment");
265 std::initializer_list<VPValue *>
Operands,
270 assert(isFPMathOp() &&
"this op can't take fast-math flags");
281 return State.
get(
this, 0, OnlyFirstLaneUsed);
287 if (
auto *
I = dyn_cast<Instruction>(Res))
297 case Instruction::ICmp: {
302 case Instruction::Select: {
324 {VIVElem0, ScalarTC},
nullptr, Name);
414 return State.
get(
this, 0,
true);
418 auto *PhiR = cast<VPReductionPHIRecipe>(
getOperand(0));
419 auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
428 Type *PhiTy = OrigPhi->getType();
430 for (
unsigned Part = 0; Part < State.
UF; ++Part)
431 RdxParts[Part] = State.
get(LoopExitingDef, Part, PhiR->isInLoop());
439 for (
unsigned Part = 0; Part < State.
UF; ++Part)
440 RdxParts[Part] = Builder.
CreateTrunc(RdxParts[Part], RdxVecTy);
443 Value *ReducedPartRdx = RdxParts[0];
446 if (PhiR->isOrdered()) {
447 ReducedPartRdx = RdxParts[State.
UF - 1];
452 for (
unsigned Part = 1; Part < State.
UF; ++Part) {
453 Value *RdxPart = RdxParts[Part];
454 if (
Op != Instruction::ICmp &&
Op != Instruction::FCmp)
460 ReducedPartRdx =
createAnyOfOp(Builder, ReductionStartValue, RK,
461 ReducedPartRdx, RdxPart);
463 ReducedPartRdx =
createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
469 if (State.
VF.
isVector() && !PhiR->isInLoop()) {
484 ReducedPartRdx,
SI->getPointerOperand(),
SI->getAlign());
488 return ReducedPartRdx;
496bool VPInstruction::isFPMathOp()
const {
499 return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
500 Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
501 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
502 Opcode == Instruction::FCmp || Opcode == Instruction::Select;
511 "Recipe not a FPMathOp but has fast-math flags?");
514 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
515 Value *GeneratedValue = generateInstruction(State, Part);
518 assert(GeneratedValue &&
"generateInstruction must produce a value");
521 State.
set(
this, GeneratedValue, Part, !IsVector);
524 "scalar value but not only first lane used");
536 case Instruction::ICmp:
548#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
556 O << Indent <<
"EMIT ";
568 O <<
"combined load";
571 O <<
"combined store";
574 O <<
"active lane mask";
577 O <<
"first-order splice";
580 O <<
"branch-on-cond";
583 O <<
"TC > VF ? TC - VF : 0";
589 O <<
"branch-on-count";
592 O <<
"compute-reduction-result";
611 assert(!isa<DbgInfoIntrinsic>(CI) &&
612 "DbgInfoIntrinsic should have been dropped during VPlan construction");
619 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
640 Arg = State.
get(
I.value(), Part);
652 assert(VectorF &&
"Can't retrieve vector intrinsic.");
655 assert(Variant !=
nullptr &&
"Can't create vector function.");
661 CI.getOperandBundlesAsDefs(OpBundles);
664 if (isa<FPMathOperator>(V))
665 V->copyFastMathFlags(&CI);
667 if (!V->getType()->isVoidTy())
668 State.
set(
this, V, Part);
673#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
676 O << Indent <<
"WIDEN-CALL ";
679 if (CI->getType()->isVoidTy())
686 O <<
"call @" << CI->getCalledFunction()->getName() <<
"(";
690 if (VectorIntrinsicID)
691 O <<
" (using vector intrinsic)";
693 O <<
" (using library function";
695 O <<
": " << Variant->
getName();
702 O << Indent <<
"WIDEN-SELECT ";
724 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
729 State.
set(
this, Sel, Part);
734VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy(
745#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
748 case OperationType::Cmp:
751 case OperationType::DisjointOp:
755 case OperationType::PossiblyExactOp:
759 case OperationType::OverflowingBinOp:
765 case OperationType::FPMathOp:
768 case OperationType::GEPOp:
772 case OperationType::NonNegOp:
776 case OperationType::Other:
788 case Instruction::Call:
789 case Instruction::Br:
790 case Instruction::PHI:
791 case Instruction::GetElementPtr:
792 case Instruction::Select:
794 case Instruction::UDiv:
795 case Instruction::SDiv:
796 case Instruction::SRem:
797 case Instruction::URem:
798 case Instruction::Add:
799 case Instruction::FAdd:
800 case Instruction::Sub:
801 case Instruction::FSub:
802 case Instruction::FNeg:
803 case Instruction::Mul:
804 case Instruction::FMul:
805 case Instruction::FDiv:
806 case Instruction::FRem:
807 case Instruction::Shl:
808 case Instruction::LShr:
809 case Instruction::AShr:
810 case Instruction::And:
811 case Instruction::Or:
812 case Instruction::Xor: {
814 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
821 if (
auto *VecOp = dyn_cast<Instruction>(V))
825 State.
set(
this, V, Part);
831 case Instruction::Freeze: {
832 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
836 State.
set(
this, Freeze, Part);
840 case Instruction::ICmp:
841 case Instruction::FCmp: {
843 bool FCmp = Opcode == Instruction::FCmp;
844 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
857 State.
set(
this,
C, Part);
873 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
876 "inferred type and type from generated instructions do not match");
881#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
884 O << Indent <<
"WIDEN ";
899 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
900 if (Part > 0 &&
Op->isLiveIn()) {
902 State.
set(
this, State.
get(
this, 0), Part);
907 State.
set(
this, Cast, Part);
912#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
915 O << Indent <<
"WIDEN-CAST ";
934 auto *ValVTy = cast<VectorType>(Val->
getType());
939 "Induction Step must be an integer or FP");
947 Type *InitVecValSTy =
957 InitVec = Builder.
CreateAdd(InitVec, StartIdxSplat);
963 return Builder.
CreateAdd(Val, Step,
"induction");
967 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
968 "Binary Opcode should be specified for FP induction");
970 InitVec = Builder.
CreateFAdd(InitVec, StartIdxSplat);
974 return Builder.
CreateBinOp(BinOp, Val, MulOp,
"induction");
981 : ConstantFP::get(Ty,
C);
989 return B.CreateUIToFP(RuntimeVF, FTy);
993 assert(!State.
Instance &&
"Int or FP induction being replicated.");
999 assert(IV->
getType() ==
ID.getStartValue()->getType() &&
"Types must match");
1004 Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
1008 if (
ID.getInductionBinOp() && isa<FPMathOperator>(
ID.getInductionBinOp()))
1014 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
1015 "Expected either an induction phi-node or a truncate of it!");
1018 auto CurrIP = Builder.
saveIP();
1021 if (isa<TruncInst>(EntryVal)) {
1022 assert(Start->getType()->isIntegerTy() &&
1023 "Truncation requires an integer type");
1024 auto *TruncType = cast<IntegerType>(EntryVal->
getType());
1026 Start = Builder.
CreateCast(Instruction::Trunc, Start, TruncType);
1032 SplatStart, Zero, Step,
ID.getInductionOpcode(), State.
VF, State.
Builder);
1039 AddOp = Instruction::Add;
1040 MulOp = Instruction::Mul;
1042 AddOp =
ID.getInductionOpcode();
1043 MulOp = Instruction::FMul;
1061 Value *SplatVF = isa<Constant>(
Mul)
1072 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1073 State.
set(
this, LastInduction, Part);
1075 if (isa<TruncInst>(EntryVal))
1078 LastInduction = cast<Instruction>(
1079 Builder.
CreateBinOp(AddOp, LastInduction, SplatVF,
"step.add"));
1083 LastInduction->
setName(
"vec.ind.next");
1093#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1096 O << Indent <<
"WIDEN-INDUCTION";
1100 O <<
" +\n" << Indent <<
"\" ";
1118 return StartC && StartC->isZero() && StepC && StepC->isOne();
1121#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1126 O << Indent <<
"= DERIVED-IV ";
1150 assert(BaseIVTy == Step->
getType() &&
"Types of BaseIV and Step must match!");
1157 AddOp = Instruction::Add;
1158 MulOp = Instruction::Mul;
1160 AddOp = InductionOpcode;
1161 MulOp = Instruction::FMul;
1170 Type *VecIVTy =
nullptr;
1171 Value *UnitStepVec =
nullptr, *SplatStep =
nullptr, *SplatIV =
nullptr;
1180 unsigned StartPart = 0;
1181 unsigned EndPart = State.
UF;
1182 unsigned StartLane = 0;
1186 EndPart = StartPart + 1;
1187 StartLane = State.
Instance->Lane.getKnownLane();
1188 EndLane = StartLane + 1;
1190 for (
unsigned Part = StartPart; Part < EndPart; ++Part) {
1195 auto *InitVec = Builder.
CreateAdd(SplatStartIdx, UnitStepVec);
1200 State.
set(
this,
Add, Part);
1209 for (
unsigned Lane = StartLane; Lane < EndLane; ++Lane) {
1215 "Expected StartIdx to be folded to a constant when VF is not "
1224#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1229 O <<
" = SCALAR-STEPS ";
1243 if (areAllOperandsInvariant()) {
1263 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1265 State.
set(
this, EntryPart, Part);
1276 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1279 auto *
Ptr = isPointerLoopInvariant()
1288 if (isIndexLoopInvariant(
I - 1))
1299 "NewGEP is not a pointer vector");
1300 State.
set(
this, NewGEP, Part);
1306#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1309 O << Indent <<
"WIDEN-GEP ";
1310 O << (isPointerLoopInvariant() ?
"Inv" :
"Var");
1312 O <<
"[" << (isIndexLoopInvariant(
I) ?
"Inv" :
"Var") <<
"]";
1316 O <<
" = getelementptr";
1323 auto &Builder = State.
Builder;
1325 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1327 Value *PartPtr =
nullptr;
1333 ?
DL.getIndexType(IndexedTy->getPointerTo())
1336 bool InBounds = isInBounds();
1345 ConstantInt::get(IndexTy, -(int64_t)Part), RunTimeVF);
1348 Builder.
CreateSub(ConstantInt::get(IndexTy, 1), RunTimeVF);
1349 PartPtr = Builder.
CreateGEP(IndexedTy,
Ptr, NumElt,
"", InBounds);
1350 PartPtr = Builder.
CreateGEP(IndexedTy, PartPtr, LastLane,
"", InBounds);
1353 PartPtr = Builder.
CreateGEP(IndexedTy,
Ptr, Increment,
"", InBounds);
1356 State.
set(
this, PartPtr, Part,
true);
1360#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1365 O <<
" = vector-pointer ";
1392 for (
unsigned In = 0; In < NumIncoming; ++In) {
1393 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1408 for (
unsigned Part = 0; Part < State.
UF; ++Part)
1409 State.
set(
this, Entry[Part], Part);
1412#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1415 O << Indent <<
"BLEND ";
1435 O << Indent <<
"REDUCE ";
1450 O <<
" (with final reduction value stored in invariant address sank "
1459 if (
auto *PredR = dyn_cast<VPPredInstPHIRecipe>(U))
1460 return any_of(PredR->users(), [PredR](
const VPUser *U) {
1461 return !U->usesScalars(PredR);
1467#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1470 O << Indent << (IsUniform ?
"CLONE " :
"REPLICATE ");
1479 O <<
"@" << CB->getCalledFunction()->getName() <<
"(";
1502 return C->isDefinedOutsideVectorRegions() ||
1503 isa<VPDerivedIVRecipe>(
C->getOperand(0)) ||
1504 isa<VPCanonicalIVPHIRecipe>(
C->getOperand(0));
1509 "Codegen only implemented for first lane.");
1511 case Instruction::SExt:
1512 case Instruction::ZExt:
1513 case Instruction::Trunc: {
1525 for (
unsigned Part = 0; Part != State.
UF; ++Part) {
1529 if (Part > 0 && IsUniformAcrossVFsAndUFs)
1532 Res = generate(State, Part);
1537#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1540 O << Indent <<
"SCALAR-CAST ";
1544 O <<
" to " << *ResultTy;
1549 assert(State.
Instance &&
"Branch on Mask works only on single instance.");
1551 unsigned Part = State.
Instance->Part;
1552 unsigned Lane = State.
Instance->Lane.getKnownLane();
1554 Value *ConditionBit =
nullptr;
1557 ConditionBit = State.
get(BlockInMask, Part);
1567 assert(isa<UnreachableInst>(CurrentTerminator) &&
1568 "Expected to replace unreachable terminator with conditional branch.");
1575 assert(State.
Instance &&
"Predicated instruction PHI works per instance.");
1580 assert(PredicatingBB &&
"Predicated block has no single predecessor.");
1582 "operand must be VPReplicateRecipe");
1590 unsigned Part = State.
Instance->Part;
1598 State.
reset(
this, VPhi, Part);
1600 State.
set(
this, VPhi, Part);
1609 Phi->addIncoming(ScalarPredInst, PredicatedBB);
1620#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1623 O << Indent <<
"PHI-PREDICATED-INSTRUCTION ";
1631 O << Indent <<
"WIDEN ";
1651 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1652 State.
set(
this, EntryPart, Part,
true);
1655#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1658 O << Indent <<
"EMIT ";
1660 O <<
" = CANONICAL-INDUCTION ";
1680 return StepC && StepC->
isOne();
1684 return IsScalarAfterVectorization &&
1688#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1691 O << Indent <<
"EMIT ";
1693 O <<
" = WIDEN-POINTER-INDUCTION ";
1695 O <<
", " << *IndDesc.
getStep();
1707 "Same SCEV expanded multiple times");
1709 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1710 State.
set(
this, Res, {Part, 0});
1713#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1716 O << Indent <<
"EMIT ";
1718 O <<
" = EXPAND SCEV " << *Expr;
1730 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
1737 Value *CanonicalVectorIV = Builder.
CreateAdd(VStart, VStep,
"vec.iv");
1738 State.
set(
this, CanonicalVectorIV, Part);
1742#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1745 O << Indent <<
"EMIT ";
1747 O <<
" = WIDEN-CANONICAL-INDUCTION ";
1753 auto &Builder = State.
Builder;
1758 ? VectorInit->getType()
1764 auto *One = ConstantInt::get(IdxTy, 1);
1768 auto *LastIdx = Builder.
CreateSub(RuntimeVF, One);
1777 State.
set(
this, EntryPart, 0);
1780#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1783 O << Indent <<
"FIRST-ORDER-RECURRENCE-PHI ";
1791 auto &Builder = State.
Builder;
1802 bool ScalarPHI = State.
VF.
isScalar() || IsInLoop;
1808 "recipe must be in the vector loop header");
1809 unsigned LastPartForNewPhi =
isOrdered() ? 1 : State.
UF;
1810 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1813 State.
set(
this, EntryPart, Part, IsInLoop);
1818 Value *Iden =
nullptr;
1844 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1845 Value *EntryPart = State.
get(
this, Part, IsInLoop);
1848 Value *StartVal = (Part == 0) ? StartV : Iden;
1849 cast<PHINode>(EntryPart)->addIncoming(StartVal, VectorPH);
1853#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1856 O << Indent <<
"WIDEN-REDUCTION-PHI ";
1866 "Non-native vplans are not expected to have VPWidenPHIRecipes.");
1871 State.
set(
this, VecPhi, 0);
1874#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1877 O << Indent <<
"WIDEN-PHI ";
1899 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
1905 State.
set(
this, EntryPart, Part);
1909#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1912 O << Indent <<
"ACTIVE-LANE-MASK-PHI ";
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
mir Rename Register Operands
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static Value * getStepVector(Value *Val, Value *StartIdx, Value *Step, Instruction::BinaryOps BinOp, ElementCount VF, IRBuilderBase &Builder)
This function adds (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step,...
static bool isUniformAcrossVFsAndUFs(VPScalarCastRecipe *C)
Checks if C is uniform across all VFs and UFs.
static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)
A helper function that returns an integer or floating-point constant with value C.
static Value * getRuntimeVFAsFloat(IRBuilderBase &B, Type *FTy, ElementCount VF)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, BasicBlock::iterator InsertBefore)
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
@ ICMP_ULT
unsigned less than
static StringRef getPredicateName(Predicate P)
This is the shared class of boolean and integer constants.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
constexpr bool isVector() const
One or more elements.
constexpr bool isScalar() const
Exactly one element.
Convenience struct for specifying and reasoning about fast-math flags.
void setAllowContract(bool B=true)
bool noSignedZeros() const
void setAllowReciprocal(bool B=true)
bool allowReciprocal() const
void print(raw_ostream &O) const
Print fast-math flags to O.
void setNoSignedZeros(bool B=true)
bool allowReassoc() const
Flag queries.
void setNoNaNs(bool B=true)
void setAllowReassoc(bool B=true)
Flag setters.
void setApproxFunc(bool B=true)
void setNoInfs(bool B=true)
bool allowContract() const
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Common base class shared among various IRBuilders.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")
Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
ConstantInt * getTrue()
Get the constant value for i1 true.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateFreeze(Value *V, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
BasicBlock * GetInsertBlock() const
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="")
InsertPoint saveIP() const
Returns the current insert point.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Value * CreateNAryOp(unsigned Opc, ArrayRef< Value * > Ops, const Twine &Name="", MDNode *FPMathTag=nullptr)
Create either a UnaryOperator or BinaryOperator depending on Opc.
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
LLVMContext & getContext() const
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
void restoreIP(InsertPoint IP)
Sets the current insert point to a previously-saved location.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_IntInduction
Integer induction variable. Step = C.
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
BlockT * getHeader() const
void print(raw_ostream &OS, const SlotIndexes *=nullptr, bool IsStandalone=true) const
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr, BasicBlock::iterator InsertBefore)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
FastMathFlags getFastMathFlags() const
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
unsigned getOpcode() const
Type * getRecurrenceType() const
Returns the type of the recurrence.
TrackingVH< Value > getRecurrenceStartValue() const
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
RecurKind getRecurrenceKind() const
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const
Returns identity corresponding to the RecurrenceKind.
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
This class uses information about analyze scalars to rewrite expressions in canonical form.
Type * getType() const
Return the LLVM type of this SCEV expression.
This class provides computation of slot numbers for LLVM Assembly writing.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Value handle that tracks a Value across RAUW.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Value * getOperand(unsigned i) const
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
void insert(VPRecipeBase *Recipe, iterator InsertPt)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPRegionBlock * getParent()
size_t getNumSuccessors() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
VPValue * getMask() const
Return the mask used by this recipe.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step) const
Check if the induction described by Kind, /p Start and Step is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
unsigned getVPDefID() const
VPCanonicalIVPHIRecipe * getCanonicalIV() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
VPValue * getStartValue() const
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
unsigned getOpcode() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
void execute(VPTransformState &State) override
Generate the instruction.
static VPLane getLastLaneForVF(const ElementCount &VF)
static VPLane getFirstLane()
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the VPLiveOut to O.
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Class to record LLVM IR flag for a recipe along with it.
NonNegFlagsTy NonNegFlags
void setFlags(Instruction *I) const
Set the IR flags for I.
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
DisjointFlagsTy DisjointFlags
bool hasNoUnsignedWrap() const
void printFlags(raw_ostream &O) const
CmpInst::Predicate getPredicate() const
bool hasNoSignedWrap() const
FastMathFlags getFastMathFlags() const
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCondOp() const
The VPValue of the condition for the block.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
VPScalarCastRecipe is a recipe to create scalar cast instructions.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
unsigned getNumOperands() const
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
friend class VPInstruction
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
void execute(VPTransformState &State) override
Produce widened copies of the cast.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
VPValue * getStepValue()
Returns the step value of the induction.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isStore() const
Returns true if this recipe is a store.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool onlyScalarsGenerated(bool IsScalable)
Returns true if only scalar values will be generated.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
StringRef getName() const
Return a constant reference to the value's name.
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
base_list_type::iterator iterator
iterator erase(iterator where)
pointer remove(iterator &IT)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
bool onlyFirstPartUsed(const VPValue *Def)
Returns true if only the first part of Def is used.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,...
Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)
Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
Value * createAnyOfOp(IRBuilderBase &Builder, Value *StartVal, RecurKind RK, Value *Left, Value *Right)
See RecurrenceDescriptor::isAnyOfPattern for a description of the pattern we are trying to match.
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
Value * createTargetReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, PHINode *OrigPhi=nullptr)
Create a generic target reduction using a recurrence descriptor Desc The target is queried to determi...
void execute(VPTransformState &State) override
Generate the phi nodes.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
bool isInvariantCond() const
VPValue * getCond() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.