43#define LV_NAME "loop-vectorize"
44#define DEBUG_TYPE LV_NAME
49 return cast<VPInterleaveRecipe>(
this)->getNumStoreOperands() > 0;
50 case VPWidenStoreEVLSC:
57 return !cast<VPWidenCallRecipe>(
this)
58 ->getCalledScalarFunction()
60 case VPBranchOnMaskSC:
61 case VPScalarIVStepsSC:
66 case VPWidenCanonicalIVSC:
69 case VPWidenIntOrFpInductionSC:
70 case VPWidenLoadEVLSC:
74 case VPWidenSelectSC: {
78 assert((!
I || !
I->mayWriteToMemory()) &&
79 "underlying instruction may write to memory");
89 case VPWidenLoadEVLSC:
94 ->mayReadFromMemory();
96 return !cast<VPWidenCallRecipe>(
this)
97 ->getCalledScalarFunction()
99 case VPBranchOnMaskSC:
100 case VPPredInstPHISC:
101 case VPScalarIVStepsSC:
102 case VPWidenStoreEVLSC:
107 case VPWidenCanonicalIVSC:
110 case VPWidenIntOrFpInductionSC:
113 case VPWidenSelectSC: {
117 assert((!
I || !
I->mayReadFromMemory()) &&
118 "underlying instruction may read from memory");
129 case VPPredInstPHISC:
132 case VPInstructionSC:
133 switch (cast<VPInstruction>(
this)->
getOpcode()) {
134 case Instruction::Or:
135 case Instruction::ICmp:
136 case Instruction::Select:
148 case VPWidenCallSC: {
149 Function *Fn = cast<VPWidenCallRecipe>(
this)->getCalledScalarFunction();
154 case VPScalarIVStepsSC:
155 case VPWidenCanonicalIVSC:
158 case VPWidenIntOrFpInductionSC:
160 case VPWidenPointerInductionSC:
162 case VPWidenSelectSC: {
166 assert((!
I || !
I->mayHaveSideEffects()) &&
167 "underlying instruction has side-effects");
172 case VPWidenLoadEVLSC:
174 case VPWidenStoreEVLSC:
179 "mayHaveSideffects result for ingredient differs from this "
182 case VPReplicateSC: {
183 auto *R = cast<VPReplicateRecipe>(
this);
184 return R->getUnderlyingInstr()->mayHaveSideEffects();
199 "the middle block must not have any successors");
205#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
216 assert(!Parent &&
"Recipe already in some VPBasicBlock");
218 "Insertion position not in any VPBasicBlock");
224 assert(!Parent &&
"Recipe already in some VPBasicBlock");
230 assert(!Parent &&
"Recipe already in some VPBasicBlock");
232 "Insertion position not in any VPBasicBlock");
259 assert(OpType == OperationType::FPMathOp &&
260 "recipe doesn't have fast math flags");
278 assert(Opcode == Instruction::ICmp &&
279 "only ICmp predicates supported at the moment");
283 std::initializer_list<VPValue *>
Operands,
288 assert(isFPMathOp() &&
"this op can't take fast-math flags");
291bool VPInstruction::doesGeneratePerAllLanes()
const {
295bool VPInstruction::canGenerateScalarForFirstLane()
const {
318 "only PtrAdd opcodes are supported for now");
332 if (
auto *
I = dyn_cast<Instruction>(Res))
342 case Instruction::ICmp: {
347 case Instruction::Select: {
369 {VIVElem0, ScalarTC},
nullptr, Name);
394 return State.
get(
this, 0,
true);
407 assert(AVL->getType()->isIntegerTy() &&
408 "Requested vector length should be an integer.");
416 {AVL, VFArg, State.Builder.getTrue()});
421 assert(Part == 0 &&
"No unrolling expected for predicated vectorization.");
426 Value *EVL = GetEVL(State, AVL);
487 return State.
get(
this, 0,
true);
491 auto *PhiR = cast<VPReductionPHIRecipe>(
getOperand(0));
492 auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
499 Type *PhiTy = OrigPhi->getType();
501 for (
unsigned Part = 0; Part < State.
UF; ++Part)
502 RdxParts[Part] = State.
get(LoopExitingDef, Part, PhiR->isInLoop());
510 for (
unsigned Part = 0; Part < State.
UF; ++Part)
511 RdxParts[Part] = Builder.
CreateTrunc(RdxParts[Part], RdxVecTy);
514 Value *ReducedPartRdx = RdxParts[0];
517 Op = Instruction::Or;
519 if (PhiR->isOrdered()) {
520 ReducedPartRdx = RdxParts[State.
UF - 1];
525 for (
unsigned Part = 1; Part < State.
UF; ++Part) {
526 Value *RdxPart = RdxParts[Part];
527 if (
Op != Instruction::ICmp &&
Op != Instruction::FCmp)
531 ReducedPartRdx =
createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
554 ReducedPartRdx,
SI->getPointerOperand(),
SI->getAlign());
558 return ReducedPartRdx;
562 return State.
get(
this, 0,
true);
565 unsigned Offset = CI->getZExtValue();
566 assert(
Offset > 0 &&
"Offset from end must be positive");
570 "invalid offset to extract from");
580 if (isa<ExtractElementInst>(Res))
591 "can only generate first lane for PtrAdd");
607bool VPInstruction::isFPMathOp()
const {
610 return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
611 Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
612 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
613 Opcode == Instruction::FCmp || Opcode == Instruction::Select;
622 "Recipe not a FPMathOp but has fast-math flags?");
626 bool GeneratesPerFirstLaneOnly =
627 canGenerateScalarForFirstLane() &&
629 bool GeneratesPerAllLanes = doesGeneratePerAllLanes();
631 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
632 if (GeneratesPerAllLanes) {
634 Lane != NumLanes; ++Lane) {
636 assert(GeneratedValue &&
"generatePerLane must produce a value");
642 if (Part != 0 && OnlyFirstPartUsed &&
hasResult()) {
643 Value *Part0 = State.
get(
this, 0, GeneratesPerFirstLaneOnly);
644 State.
set(
this, Part0, Part,
645 GeneratesPerFirstLaneOnly);
649 Value *GeneratedValue = generatePerPart(State, Part);
652 assert(GeneratedValue &&
"generatePerPart must produce a value");
654 !GeneratesPerFirstLaneOnly ||
656 "scalar value but not only first lane defined");
657 State.
set(
this, GeneratedValue, Part,
658 GeneratesPerFirstLaneOnly);
670 case Instruction::ICmp:
692 case Instruction::ICmp:
693 case Instruction::Select:
703#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
711 O << Indent <<
"EMIT ";
723 O <<
"combined load";
726 O <<
"combined store";
729 O <<
"active lane mask";
732 O <<
"EXPLICIT-VECTOR-LENGTH";
735 O <<
"first-order splice";
738 O <<
"branch-on-cond";
741 O <<
"TC > VF ? TC - VF : 0";
747 O <<
"branch-on-count";
750 O <<
"extract-from-end";
753 O <<
"compute-reduction-result";
779 "DbgInfoIntrinsic should have been dropped during VPlan construction");
786 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
807 Arg = State.
get(
I.value(), Part);
819 assert(VectorF &&
"Can't retrieve vector intrinsic.");
822 assert(Variant !=
nullptr &&
"Can't create vector function.");
830 CI->getOperandBundlesAsDefs(OpBundles);
834 if (isa<FPMathOperator>(V))
835 V->copyFastMathFlags(CI);
837 if (!V->getType()->isVoidTy())
838 State.
set(
this, V, Part);
843#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
846 O << Indent <<
"WIDEN-CALL ";
856 O <<
"call @" << CalledFn->
getName() <<
"(";
862 if (VectorIntrinsicID)
863 O <<
" (using vector intrinsic)";
865 O <<
" (using library function";
867 O <<
": " << Variant->
getName();
874 O << Indent <<
"WIDEN-SELECT ";
896 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
901 State.
set(
this, Sel, Part);
906VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy(
917#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
920 case OperationType::Cmp:
923 case OperationType::DisjointOp:
927 case OperationType::PossiblyExactOp:
931 case OperationType::OverflowingBinOp:
937 case OperationType::FPMathOp:
940 case OperationType::GEPOp:
944 case OperationType::NonNegOp:
948 case OperationType::Other:
960 case Instruction::Call:
961 case Instruction::Br:
962 case Instruction::PHI:
963 case Instruction::GetElementPtr:
964 case Instruction::Select:
966 case Instruction::UDiv:
967 case Instruction::SDiv:
968 case Instruction::SRem:
969 case Instruction::URem:
970 case Instruction::Add:
971 case Instruction::FAdd:
972 case Instruction::Sub:
973 case Instruction::FSub:
974 case Instruction::FNeg:
975 case Instruction::Mul:
976 case Instruction::FMul:
977 case Instruction::FDiv:
978 case Instruction::FRem:
979 case Instruction::Shl:
980 case Instruction::LShr:
981 case Instruction::AShr:
982 case Instruction::And:
983 case Instruction::Or:
984 case Instruction::Xor: {
986 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
993 if (
auto *VecOp = dyn_cast<Instruction>(V))
997 State.
set(
this, V, Part);
1003 case Instruction::Freeze: {
1004 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1008 State.
set(
this, Freeze, Part);
1012 case Instruction::ICmp:
1013 case Instruction::FCmp: {
1015 bool FCmp = Opcode == Instruction::FCmp;
1016 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1029 State.
set(
this,
C, Part);
1045 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1048 "inferred type and type from generated instructions do not match");
1053#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1056 O << Indent <<
"WIDEN ";
1066 auto &Builder = State.
Builder;
1071 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1072 if (Part > 0 &&
Op->isLiveIn()) {
1074 State.
set(
this, State.
get(
this, 0), Part);
1079 State.
set(
this, Cast, Part);
1084#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1087 O << Indent <<
"WIDEN-CAST ";
1106 auto *ValVTy = cast<VectorType>(Val->
getType());
1111 "Induction Step must be an integer or FP");
1119 Type *InitVecValSTy =
1129 InitVec = Builder.
CreateAdd(InitVec, StartIdxSplat);
1134 Step = Builder.
CreateMul(InitVec, Step);
1135 return Builder.
CreateAdd(Val, Step,
"induction");
1139 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
1140 "Binary Opcode should be specified for FP induction");
1142 InitVec = Builder.
CreateFAdd(InitVec, StartIdxSplat);
1146 return Builder.
CreateBinOp(BinOp, Val, MulOp,
"induction");
1153 : ConstantFP::get(Ty,
C);
1161 return B.CreateUIToFP(RuntimeVF, FTy);
1165 assert(!State.
Instance &&
"Int or FP induction being replicated.");
1171 assert(IV->
getType() ==
ID.getStartValue()->getType() &&
"Types must match");
1176 Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
1180 if (
ID.getInductionBinOp() && isa<FPMathOperator>(
ID.getInductionBinOp()))
1186 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
1187 "Expected either an induction phi-node or a truncate of it!");
1190 auto CurrIP = Builder.
saveIP();
1193 if (isa<TruncInst>(EntryVal)) {
1194 assert(Start->getType()->isIntegerTy() &&
1195 "Truncation requires an integer type");
1196 auto *TruncType = cast<IntegerType>(EntryVal->
getType());
1198 Start = Builder.
CreateCast(Instruction::Trunc, Start, TruncType);
1204 SplatStart, Zero, Step,
ID.getInductionOpcode(), State.
VF, State.
Builder);
1211 AddOp = Instruction::Add;
1212 MulOp = Instruction::Mul;
1214 AddOp =
ID.getInductionOpcode();
1215 MulOp = Instruction::FMul;
1233 Value *SplatVF = isa<Constant>(
Mul)
1244 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1245 State.
set(
this, LastInduction, Part);
1247 if (isa<TruncInst>(EntryVal))
1250 LastInduction = cast<Instruction>(
1251 Builder.
CreateBinOp(AddOp, LastInduction, SplatVF,
"step.add"));
1255 LastInduction->
setName(
"vec.ind.next");
1265#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1268 O << Indent <<
"WIDEN-INDUCTION";
1272 O <<
" +\n" << Indent <<
"\" ";
1290 auto *CanIV = cast<VPCanonicalIVPHIRecipe>(&*
getParent()->begin());
1291 return StartC && StartC->isZero() && StepC && StepC->isOne() &&
1295#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1300 O << Indent <<
"= DERIVED-IV ";
1324 assert(BaseIVTy == Step->
getType() &&
"Types of BaseIV and Step must match!");
1331 AddOp = Instruction::Add;
1332 MulOp = Instruction::Mul;
1334 AddOp = InductionOpcode;
1335 MulOp = Instruction::FMul;
1344 Type *VecIVTy =
nullptr;
1345 Value *UnitStepVec =
nullptr, *SplatStep =
nullptr, *SplatIV =
nullptr;
1354 unsigned StartPart = 0;
1355 unsigned EndPart = State.
UF;
1356 unsigned StartLane = 0;
1360 EndPart = StartPart + 1;
1361 StartLane = State.
Instance->Lane.getKnownLane();
1362 EndLane = StartLane + 1;
1364 for (
unsigned Part = StartPart; Part < EndPart; ++Part) {
1369 auto *InitVec = Builder.
CreateAdd(SplatStartIdx, UnitStepVec);
1374 State.
set(
this,
Add, Part);
1383 for (
unsigned Lane = StartLane; Lane < EndLane; ++Lane) {
1389 "Expected StartIdx to be folded to a constant when VF is not "
1398#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1403 O <<
" = SCALAR-STEPS ";
1417 if (areAllOperandsInvariant()) {
1437 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1439 State.
set(
this, EntryPart, Part);
1450 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1453 auto *
Ptr = isPointerLoopInvariant()
1462 if (isIndexLoopInvariant(
I - 1))
1473 "NewGEP is not a pointer vector");
1474 State.
set(
this, NewGEP, Part);
1480#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1483 O << Indent <<
"WIDEN-GEP ";
1484 O << (isPointerLoopInvariant() ?
"Inv" :
"Var");
1486 O <<
"[" << (isIndexLoopInvariant(
I) ?
"Inv" :
"Var") <<
"]";
1490 O <<
" = getelementptr";
1497 auto &Builder = State.
Builder;
1499 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1501 Value *PartPtr =
nullptr;
1507 ?
DL.getIndexType(IndexedTy->getPointerTo())
1510 bool InBounds = isInBounds();
1519 ConstantInt::get(IndexTy, -(int64_t)Part), RunTimeVF);
1522 Builder.
CreateSub(ConstantInt::get(IndexTy, 1), RunTimeVF);
1523 PartPtr = Builder.
CreateGEP(IndexedTy,
Ptr, NumElt,
"", InBounds);
1524 PartPtr = Builder.
CreateGEP(IndexedTy, PartPtr, LastLane,
"", InBounds);
1527 PartPtr = Builder.
CreateGEP(IndexedTy,
Ptr, Increment,
"", InBounds);
1530 State.
set(
this, PartPtr, Part,
true);
1534#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1539 O <<
" = vector-pointer ";
1567 for (
unsigned In = 0; In < NumIncoming; ++In) {
1568 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1583 for (
unsigned Part = 0; Part < State.
UF; ++Part)
1584 State.
set(
this, Entry[Part], Part, OnlyFirstLaneUsed);
1587#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1590 O << Indent <<
"BLEND ";
1618 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1643 PrevInChain = NewRed;
1650 NewRed, PrevInChain);
1651 }
else if (IsOrdered)
1652 NextInChain = NewRed;
1656 State.
set(
this, NextInChain, Part,
true);
1660#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1663 O << Indent <<
"REDUCE ";
1678 O <<
" (with final reduction value stored in invariant address sank "
1687 if (
auto *PredR = dyn_cast<VPPredInstPHIRecipe>(U))
1688 return any_of(PredR->users(), [PredR](
const VPUser *U) {
1689 return !U->usesScalars(PredR);
1695#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1698 O << Indent << (IsUniform ?
"CLONE " :
"REPLICATE ");
1707 O <<
"@" << CB->getCalledFunction()->getName() <<
"(";
1730 return C->isDefinedOutsideVectorRegions() ||
1731 isa<VPDerivedIVRecipe>(
C->getOperand(0)) ||
1732 isa<VPCanonicalIVPHIRecipe>(
C->getOperand(0));
1737 "Codegen only implemented for first lane.");
1739 case Instruction::SExt:
1740 case Instruction::ZExt:
1741 case Instruction::Trunc: {
1753 for (
unsigned Part = 0; Part != State.
UF; ++Part) {
1757 if (Part > 0 && IsUniformAcrossVFsAndUFs)
1760 Res = generate(State, Part);
1765#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1768 O << Indent <<
"SCALAR-CAST ";
1772 O <<
" to " << *ResultTy;
1777 assert(State.
Instance &&
"Branch on Mask works only on single instance.");
1779 unsigned Part = State.
Instance->Part;
1780 unsigned Lane = State.
Instance->Lane.getKnownLane();
1782 Value *ConditionBit =
nullptr;
1785 ConditionBit = State.
get(BlockInMask, Part);
1795 assert(isa<UnreachableInst>(CurrentTerminator) &&
1796 "Expected to replace unreachable terminator with conditional branch.");
1803 assert(State.
Instance &&
"Predicated instruction PHI works per instance.");
1808 assert(PredicatingBB &&
"Predicated block has no single predecessor.");
1810 "operand must be VPReplicateRecipe");
1818 unsigned Part = State.
Instance->Part;
1826 State.
reset(
this, VPhi, Part);
1828 State.
set(
this, VPhi, Part);
1837 Phi->addIncoming(ScalarPredInst, PredicatedBB);
1848#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1851 O << Indent <<
"PHI-PREDICATED-INSTRUCTION ";
1859 O << Indent <<
"WIDEN ";
1867 O << Indent <<
"WIDEN ";
1875 O << Indent <<
"WIDEN store ";
1881 O << Indent <<
"WIDEN vp.store ";
1894 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1895 State.
set(
this, EntryPart, Part,
true);
1898#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1901 O << Indent <<
"EMIT ";
1903 O <<
" = CANONICAL-INDUCTION ";
1923 return StepC && StepC->
isOne();
1927 return IsScalarAfterVectorization &&
1931#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1934 O << Indent <<
"EMIT ";
1936 O <<
" = WIDEN-POINTER-INDUCTION ";
1938 O <<
", " << *IndDesc.
getStep();
1950 "Same SCEV expanded multiple times");
1952 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1953 State.
set(
this, Res, {Part, 0});
1956#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1959 O << Indent <<
"EMIT ";
1961 O <<
" = EXPAND SCEV " << *Expr;
1973 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
1980 Value *CanonicalVectorIV = Builder.
CreateAdd(VStart, VStep,
"vec.iv");
1981 State.
set(
this, CanonicalVectorIV, Part);
1985#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1988 O << Indent <<
"EMIT ";
1990 O <<
" = WIDEN-CANONICAL-INDUCTION ";
1996 auto &Builder = State.
Builder;
2001 ? VectorInit->getType()
2007 auto *One = ConstantInt::get(IdxTy, 1);
2011 auto *LastIdx = Builder.
CreateSub(RuntimeVF, One);
2020 State.
set(
this, EntryPart, 0);
2023#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2026 O << Indent <<
"FIRST-ORDER-RECURRENCE-PHI ";
2034 auto &Builder = State.
Builder;
2045 bool ScalarPHI = State.
VF.
isScalar() || IsInLoop;
2051 "recipe must be in the vector loop header");
2052 unsigned LastPartForNewPhi =
isOrdered() ? 1 : State.
UF;
2053 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
2056 State.
set(
this, EntryPart, Part, IsInLoop);
2061 Value *Iden =
nullptr;
2087 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
2088 Value *EntryPart = State.
get(
this, Part, IsInLoop);
2091 Value *StartVal = (Part == 0) ? StartV : Iden;
2092 cast<PHINode>(EntryPart)->addIncoming(StartVal, VectorPH);
2096#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2099 O << Indent <<
"WIDEN-REDUCTION-PHI ";
2109 "Non-native vplans are not expected to have VPWidenPHIRecipes.");
2114 State.
set(
this, VecPhi, 0);
2117#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2120 O << Indent <<
"WIDEN-PHI ";
2142 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
2148 State.
set(
this, EntryPart, Part);
2152#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2155 O << Indent <<
"ACTIVE-LANE-MASK-PHI ";
2165 assert(State.
UF == 1 &&
"Expected unroll factor 1 for VP vectorization.");
2171 State.
set(
this, EntryPart, 0,
true);
2174#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2177 O << Indent <<
"EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
mir Rename Register Operands
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static Value * getStepVector(Value *Val, Value *StartIdx, Value *Step, Instruction::BinaryOps BinOp, ElementCount VF, IRBuilderBase &Builder)
This function adds (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step,...
static bool isUniformAcrossVFsAndUFs(VPScalarCastRecipe *C)
Checks if C is uniform across all VFs and UFs.
static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)
A helper function that returns an integer or floating-point constant with value C.
static Value * getRuntimeVFAsFloat(IRBuilderBase &B, Type *FTy, ElementCount VF)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const DataLayout & getDataLayout() const
Get the data layout of the module this basic block belongs to.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
@ ICMP_ULT
unsigned less than
static StringRef getPredicateName(Predicate P)
This is the shared class of boolean and integer constants.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
constexpr bool isVector() const
One or more elements.
constexpr bool isScalar() const
Exactly one element.
Convenience struct for specifying and reasoning about fast-math flags.
void setAllowContract(bool B=true)
bool noSignedZeros() const
void setAllowReciprocal(bool B=true)
bool allowReciprocal() const
void print(raw_ostream &O) const
Print fast-math flags to O.
void setNoSignedZeros(bool B=true)
bool allowReassoc() const
Flag queries.
void setNoNaNs(bool B=true)
void setAllowReassoc(bool B=true)
Flag setters.
void setApproxFunc(bool B=true)
void setNoInfs(bool B=true)
bool allowContract() const
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
bool willReturn() const
Determine if the function will return.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
bool doesNotThrow() const
Determine if the function cannot unwind.
Type * getReturnType() const
Returns the type of the ret val.
Common base class shared among various IRBuilders.
Value * CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")
Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
ConstantInt * getTrue()
Get the constant value for i1 true.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateFreeze(Value *V, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
BasicBlock * GetInsertBlock() const
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
InsertPoint saveIP() const
Returns the current insert point.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Value * CreateNAryOp(unsigned Opc, ArrayRef< Value * > Ops, const Twine &Name="", MDNode *FPMathTag=nullptr)
Create either a UnaryOperator or BinaryOperator depending on Opc.
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
LLVMContext & getContext() const
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name="")
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
void restoreIP(InsertPoint IP)
Sets the current insert point to a previously-saved location.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_IntInduction
Integer induction variable. Step = C.
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
BlockT * getHeader() const
void print(raw_ostream &OS, const SlotIndexes *=nullptr, bool IsStandalone=true) const
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
FastMathFlags getFastMathFlags() const
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
unsigned getOpcode() const
Type * getRecurrenceType() const
Returns the type of the recurrence.
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
RecurKind getRecurrenceKind() const
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const
Returns identity corresponding to the RecurrenceKind.
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
This class uses information about analyze scalars to rewrite expressions in canonical form.
Type * getType() const
Return the LLVM type of this SCEV expression.
This class provides computation of slot numbers for LLVM Assembly writing.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Value * getOperand(unsigned i) const
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
void insert(VPRecipeBase *Recipe, iterator InsertPt)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that the first incoming value has no mask.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPRegionBlock * getParent()
size_t getNumSuccessors() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
VPValue * getMask() const
Return the mask used by this recipe.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step) const
Check if the induction described by Kind, /p Start and Step is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
unsigned getVPDefID() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
VPValue * getStartValue() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate phi for handling IV based on EVL over iterations correctly.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
unsigned getOpcode() const
bool onlyFirstPartUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
bool isVectorToScalar() const
Returns true if this VPInstruction produces a scalar value from a vector, e.g.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
void execute(VPTransformState &State) override
Generate the instruction.
static VPLane getLastLaneForVF(const ElementCount &VF)
static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset)
static VPLane getFirstLane()
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the VPLiveOut to O.
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Class to record LLVM IR flag for a recipe along with it.
NonNegFlagsTy NonNegFlags
void setFlags(Instruction *I) const
Set the IR flags for I.
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
DisjointFlagsTy DisjointFlags
bool hasNoUnsignedWrap() const
void printFlags(raw_ostream &O) const
CmpInst::Predicate getPredicate() const
bool hasNoSignedWrap() const
FastMathFlags getFastMathFlags() const
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCondOp() const
The VPValue of the condition for the block.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
void execute(VPTransformState &State) override
Generate the reduction in the loop.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
unsigned getOpcode() const
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
VPScalarCastRecipe is a recipe to create scalar cast instructions.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
This class can be used to assign names to VPValues.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
unsigned getNumOperands() const
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
friend class VPInstruction
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Function * getCalledScalarFunction() const
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
operand_range arg_operands()
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
void execute(VPTransformState &State) override
Produce widened copies of the cast.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
VPValue * getStepValue()
Returns the step value of the induction.
Type * getScalarType() const
Returns the scalar type of the induction.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool onlyScalarsGenerated(bool IsScalable)
Returns true if only scalar values will be generated.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
StringRef getName() const
Return a constant reference to the value's name.
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Type * getElementType() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
const ParentTy * getParent() const
self_iterator getIterator()
base_list_type::iterator iterator
iterator erase(iterator where)
pointer remove(iterator &IT)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
bool onlyFirstPartUsed(const VPValue *Def)
Returns true if only the first part of Def is used.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,...
Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)
Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
static bool isDbgInfoIntrinsic(Intrinsic::ID ID)
Check if ID corresponds to a debug info intrinsic.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Value * createOrderedReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, Value *Start)
Create an ordered reduction intrinsic using the given recurrence descriptor Desc.
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
Value * createTargetReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, PHINode *OrigPhi=nullptr)
Create a generic target reduction using a recurrence descriptor Desc The target is queried to determi...
void execute(VPTransformState &State) override
Generate the phi nodes.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isInvariantCond() const
VPValue * getCond() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.