49#define LV_NAME "loop-vectorize"
50#define DEBUG_TYPE LV_NAME
61 "expected function operand");
84 case VPInstructionSC: {
87 if (VPI->getOpcode() == Instruction::Load ||
90 return VPI->opcodeMayReadOrWriteFromMemory();
92 case VPInterleaveEVLSC:
95 case VPWidenStoreEVLSC:
100 ->mayWriteToMemory();
103 ->getCalledScalarFunction()
105 case VPWidenMemIntrinsicSC:
106 case VPWidenIntrinsicSC:
108 case VPActiveLaneMaskPHISC:
109 case VPCurrentIterationPHISC:
110 case VPBranchOnMaskSC:
112 case VPFirstOrderRecurrencePHISC:
113 case VPReductionPHISC:
114 case VPScalarIVStepsSC:
115 case VPPredInstPHISC:
117 case VPConditionalInductionPHISC:
120 case VPReductionEVLSC:
122 case VPVectorPointerSC:
123 case VPVectorEndPointerSC:
124 case VPWidenCanonicalIVSC:
127 case VPWidenIntOrFpInductionSC:
128 case VPWidenLoadEVLSC:
131 case VPWidenPointerInductionSC:
136 assert((!
I || !
I->mayWriteToMemory()) &&
137 "underlying instruction may write to memory");
149 case VPInstructionSC: {
154 return VPI->opcodeMayReadOrWriteFromMemory();
156 case VPWidenLoadEVLSC:
161 ->mayReadFromMemory();
164 ->getCalledScalarFunction()
165 ->onlyWritesMemory();
166 case VPWidenMemIntrinsicSC:
167 case VPWidenIntrinsicSC:
169 case VPActiveLaneMaskPHISC:
170 case VPBranchOnMaskSC:
172 case VPCurrentIterationPHISC:
173 case VPFirstOrderRecurrencePHISC:
174 case VPReductionPHISC:
175 case VPPredInstPHISC:
176 case VPScalarIVStepsSC:
177 case VPWidenStoreEVLSC:
180 case VPConditionalInductionPHISC:
183 case VPReductionEVLSC:
185 case VPVectorPointerSC:
186 case VPVectorEndPointerSC:
187 case VPWidenCanonicalIVSC:
190 case VPWidenIntOrFpInductionSC:
192 case VPWidenPointerInductionSC:
197 assert((!
I || !
I->mayReadFromMemory()) &&
198 "underlying instruction may read from memory");
211 case VPActiveLaneMaskPHISC:
213 case VPCurrentIterationPHISC:
214 case VPFirstOrderRecurrencePHISC:
215 case VPReductionPHISC:
216 case VPPredInstPHISC:
217 case VPVectorEndPointerSC:
219 case VPConditionalInductionPHISC:
221 case VPInstructionSC: {
223 if (VPI->getOpcode() == Instruction::Call ||
227 !VPI->hasFnAttr(Attribute::NoUnwind, CalleeFnAttrs) ||
228 !VPI->hasFnAttr(Attribute::WillReturn, CalleeFnAttrs);
235 case VPWidenCallSC: {
239 case VPWidenMemIntrinsicSC:
240 case VPWidenIntrinsicSC:
243 case VPReductionEVLSC:
245 case VPScalarIVStepsSC:
246 case VPVectorPointerSC:
247 case VPWidenCanonicalIVSC:
250 case VPWidenIntOrFpInductionSC:
252 case VPWidenPointerInductionSC:
257 assert((!
I || !
I->mayHaveSideEffects()) &&
258 "underlying instruction has side-effects");
261 case VPInterleaveEVLSC:
264 case VPWidenLoadEVLSC:
266 case VPWidenStoreEVLSC:
271 "mayHaveSideffects result for ingredient differs from this "
274 case VPReplicateSC: {
276 return R->getUnderlyingInstr()->mayHaveSideEffects();
287 case VPInstructionSC: {
295 case Instruction::Add:
296 case Instruction::Sub:
297 case Instruction::Mul:
298 case Instruction::GetElementPtr:
300 case Instruction::Call:
315 assert(
I.getNodeParent() &&
"Insertion position not in any VPBasicBlock");
316 I.getNodeParent()->insert(
this,
I);
321 "Insertion position not in any VPBasicBlock");
355 UI = IG->getInsertPos();
357 UI = &WidenMem->getIngredient();
363 bool ReplicatesScalable =
364 RepR && VF.
isScalable() && RepR->doesGeneratePerAllLanes();
366 if (UI && Ctx.skipCostComputation(UI, VF.
isVector()) && !ReplicatesScalable) {
370 if (Ctx.Opts.force_target_instruction_cost && RecipeCost.
isValid()) {
383 dbgs() <<
"Cost of " << RecipeCost <<
" for VF " << VF <<
": ";
405 assert(OpType == Other.OpType &&
"OpType must match");
407 case OperationType::OverflowingBinOp:
408 WrapFlags.HasNUW &= Other.WrapFlags.HasNUW;
409 WrapFlags.HasNSW &= Other.WrapFlags.HasNSW;
411 case OperationType::Trunc:
415 case OperationType::DisjointOp:
418 case OperationType::PossiblyExactOp:
419 ExactFlags.IsExact &= Other.ExactFlags.IsExact;
421 case OperationType::GEPOp:
424 case OperationType::FPMathOp:
425 case OperationType::FCmp:
426 assert((OpType != OperationType::FCmp ||
427 FCmpFlags.CmpPredStorage == Other.FCmpFlags.CmpPredStorage) &&
428 "Cannot drop CmpPredicate");
431 case OperationType::NonNegOp:
434 case OperationType::Cmp:
436 "Cannot drop CmpPredicate");
438 case OperationType::ReductionOp:
440 "Cannot change RecurKind");
442 "Cannot change IsOrdered");
444 "Cannot change IsInLoop");
447 case OperationType::Other:
455 const FastMathFlagsTy &
F = getFMFsRef();
467#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
485 Expr(Expr), ExpandInIR(ExpandInIR) {}
496 "zero-operand VPInstruction opcodes must pass explicit ResultTy");
498 [[maybe_unused]]
auto AssertOperandType = [&
Operands](
unsigned Idx,
500 if (!ExpectedTy ||
Operands.size() <= Idx)
504 "different types inferred for different operands");
519 AssertOperandType(1, Op0Ty);
523 for (
unsigned Idx = 1; Idx !=
Operands.size(); ++Idx)
524 AssertOperandType(Idx, Op0Ty);
526 case Instruction::Switch:
527 for (
unsigned Idx = 1; Idx !=
Operands.size(); ++Idx)
528 AssertOperandType(Idx, Op0Ty);
531 case Instruction::Store:
533 case Instruction::ICmp:
535 AssertOperandType(1, Op0Ty);
537 case Instruction::FCmp:
539 AssertOperandType(1, Op0Ty);
544 AssertOperandType(1, Op0Ty);
552 AssertOperandType(1, Op0Ty);
556 for (
unsigned Idx = 1; Idx !=
Operands.size(); ++Idx)
557 AssertOperandType(Idx, Op0Ty);
562 case Instruction::Select: {
564 "select condition must be bool");
566 AssertOperandType(2, Op1Ty);
569 case Instruction::InsertElement:
572 AssertOperandType(1, Op0Ty);
574 "expected integer operand");
579 AssertOperandType(1, Op0Ty);
582 assert(
Operands.size() >= 2 &&
"ExtractLane requires a lane operand and "
583 "at least one source vector operand");
587 for (
unsigned Idx = 2; Idx !=
Operands.size(); ++Idx)
588 AssertOperandType(Idx, Op1Ty);
594 "expected pointer operand");
596 "expected integer operand");
598 case Instruction::ExtractValue:
606 case Instruction::Load:
607 case Instruction::Alloca:
609 case Instruction::Call:
613 "BuildVector operands must be valid vector elements");
623 bool AllOperandsSameType =
629 if (AllOperandsSameType)
630 for (
unsigned Idx = 1; Idx !=
Operands.size(); ++Idx)
631 AssertOperandType(Idx, Op0Ty);
638 unsigned Opcode =
I->getOpcode();
658 "Set flags not supported for the provided opcode");
660 "Opcode requires specific flags to be set");
664 "number of operands does not match opcode");
678 case Instruction::Alloca:
679 case Instruction::Freeze:
680 case Instruction::Load:
694 case Instruction::ICmp:
695 case Instruction::FCmp:
696 case Instruction::ExtractElement:
697 case Instruction::Store:
710 case Instruction::InsertElement:
711 case Instruction::Select:
718 case Instruction::Call:
720 case Instruction::GetElementPtr:
721 case Instruction::PHI:
722 case Instruction::Switch:
723 case Instruction::AtomicRMW:
724 case Instruction::AtomicCmpXchg:
725 case Instruction::ExtractValue:
726 case Instruction::Fence:
748bool VPInstruction::doesGenerateSingleScalar()
const {
752 case Instruction::Freeze:
753 case Instruction::ICmp:
754 case Instruction::PHI:
755 case Instruction::Select:
772 return Instruction::Add;
774 return Instruction::FAdd;
779 bool GenerateSingleScalar) {
780 IRBuilderBase &Builder = State.
Builder;
819 case Instruction::ExtractElement: {
820 assert(GenerateSingleScalar &&
821 "Can only generate first lane for ExtractElement");
829 case Instruction::InsertElement: {
830 assert(!GenerateSingleScalar &&
831 "Cannot generate scalar value for InsertElement");
838 case Instruction::Freeze: {
842 case Instruction::FCmp:
843 case Instruction::ICmp: {
848 case Instruction::PHI: {
851 case Instruction::Select: {
886 {VIVElem0, ScalarTC},
nullptr, Name);
889 assert(GenerateSingleScalar &&
890 "Can only generate first lane for NumActiveLanes");
893 assert(VecTy->getScalarSizeInBits() == 1 &&
894 "NumActiveLanes only implemented for i1 vectors");
917 if (!
V1->getType()->isVectorTy())
925 assert(GenerateSingleScalar &&
926 "Can only generate first lane for ExplicitVectorLength");
930 "Requested vector length should be an integer.");
936 Builder.
getInt32Ty(), Intrinsic::experimental_get_vector_length,
937 {AVL, VFArg, Builder.getTrue()});
941 assert(GenerateSingleScalar &&
942 "Can only generate first lane for BranchOnCond");
948 VPBasicBlock *SecondVPSucc =
960 assert(!GenerateSingleScalar &&
961 "Cannot generate scalar value for Broadcast");
967 assert(!GenerateSingleScalar &&
968 "Cannot generate scalar value for BuildVector");
988 IRBuilderBase::FastMathFlagGuard FMFG(Builder);
999 assert(GenerateSingleScalar &&
1000 "Can only generate first lane for ComputeReductionResult");
1005 "FindIV should use min/max reduction kinds");
1009 SmallVector<Value *, 2> RdxParts(NumOperandsToReduce);
1010 for (
unsigned Part = 0; Part < NumOperandsToReduce; ++Part)
1013 IRBuilderBase::FastMathFlagGuard FMFG(Builder);
1017 Value *ReducedPartRdx = RdxParts[0];
1019 ReducedPartRdx = RdxParts[NumOperandsToReduce - 1];
1022 for (
unsigned Part = 1; Part < NumOperandsToReduce; ++Part) {
1023 Value *RdxPart = RdxParts[Part];
1025 ReducedPartRdx =
createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
1034 Builder.
CreateBinOp(Opcode, RdxPart, ReducedPartRdx,
"bin.rdx");
1048 return ReducedPartRdx;
1052 assert(GenerateSingleScalar &&
1053 "Can only generate first lane for ExtractLane and "
1054 "ExtractPenultimateElement");
1060 "invalid offset to extract from");
1065 assert(
Offset <= 1 &&
"invalid offset to extract from");
1073 assert(GenerateSingleScalar &&
"Can only generate first lane for PtrAdd");
1082 assert(!GenerateSingleScalar &&
1083 "Cannot generate scalar value for WidePtrAdd");
1090 assert(GenerateSingleScalar &&
"Can only generate first lane for AnyOf");
1097 assert(GenerateSingleScalar &&
1098 "Can only generate first lane for ExtractLane");
1100 "simplified to ExtractElement.");
1103 Value *Res =
nullptr;
1107 Value *VectorStart =
1108 Builder.
CreateMul(RuntimeVF, ConstantInt::get(IdxTy, Idx - 1));
1111 : Builder.
CreateSub(LaneToExtract, VectorStart);
1126 assert(GenerateSingleScalar &&
1127 "Can only generate first lane for FirstActiveLane");
1139 Value *Res =
nullptr;
1140 for (
int Idx = LastOpIdx;
Idx >= 0; --
Idx) {
1141 Value *TrailingZeros =
1151 Builder.
CreateMul(RuntimeVF, ConstantInt::get(Ty, Idx)),
1164 assert(GenerateSingleScalar &&
1165 "Can only generate first lane for ResumeForEpilogue");
1168 assert(!GenerateSingleScalar &&
"Cannot generate scalar value for Reverse");
1171 assert(GenerateSingleScalar &&
1172 "Can only generate first lane for ExtractLastActive");
1183 Intrinsic::experimental_vector_extract_last_active, {VTy},
1190 assert(!GenerateSingleScalar &&
1191 "Cannot generate scalar value for ExtractVectorForPart");
1196 if (Src->getType() == DstTy)
1203 assert(!GenerateSingleScalar &&
1204 "Cannot generate scalar value for StepVector");
1208 assert(GenerateSingleScalar &&
1209 "Can only generate first lane for Intrinsic");
1210 SmallVector<Value *, 2>
Args;
1237 State.
VF * Multiplier &&
1238 "stored value does not match wide element count");
1265 case Instruction::FNeg:
1266 return Ctx.TTI.getArithmeticInstrCost(Opcode, ResultTy, Ctx.CostKind);
1267 case Instruction::UDiv:
1268 case Instruction::SDiv:
1269 case Instruction::SRem:
1270 case Instruction::URem:
1271 case Instruction::Add:
1272 case Instruction::FAdd:
1273 case Instruction::Sub:
1274 case Instruction::FSub:
1275 case Instruction::Mul:
1276 case Instruction::FMul:
1277 case Instruction::FDiv:
1278 case Instruction::FRem:
1279 case Instruction::Shl:
1280 case Instruction::LShr:
1281 case Instruction::AShr:
1282 case Instruction::And:
1283 case Instruction::Or:
1284 case Instruction::Xor: {
1298 return Ctx.TTI.getArithmeticInstrCost(
1299 Opcode, ResultTy, Ctx.CostKind,
1300 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
1301 RHSInfo, Operands, CtxI, &Ctx.TLI);
1303 case Instruction::Freeze:
1310 case Instruction::ExtractValue:
1311 return Ctx.TTI.getInsertExtractValueCost(Instruction::ExtractValue,
1313 case Instruction::ICmp:
1314 case Instruction::FCmp: {
1318 return Ctx.TTI.getCmpSelInstrCost(
1320 Ctx.CostKind, {TTI::OK_AnyValue, TTI::OP_None},
1321 {TTI::OK_AnyValue, TTI::OP_None}, CtxI);
1323 case Instruction::BitCast: {
1329 case Instruction::SExt:
1330 case Instruction::ZExt:
1331 case Instruction::FPToUI:
1332 case Instruction::FPToSI:
1333 case Instruction::FPExt:
1334 case Instruction::PtrToInt:
1335 case Instruction::PtrToAddr:
1336 case Instruction::IntToPtr:
1337 case Instruction::SIToFP:
1338 case Instruction::UIToFP:
1339 case Instruction::Trunc:
1340 case Instruction::FPTrunc:
1341 case Instruction::AddrSpaceCast: {
1356 if (WidenMemoryRecipe ==
nullptr)
1360 if (!WidenMemoryRecipe->isConsecutive())
1362 if (WidenMemoryRecipe->isMasked())
1369 bool IsReverse =
false;
1371 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
1379 Recipe->getVPSingleValue()->getSingleUser());
1382 CCH = ComputeCCH(Recipe);
1386 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
1387 Opcode == Instruction::FPExt) {
1398 CCH = ComputeCCH(Recipe);
1407 return Ctx.TTI.getCastInstrCost(
1408 Opcode, ResultTy, SrcTy, CCH, Ctx.CostKind,
1411 case Instruction::Select: {
1430 (IsLogicalAnd || IsLogicalOr)) {
1433 const auto [Op1VK, Op1VP] = Ctx.getOperandInfo(Op0);
1434 const auto [Op2VK, Op2VP] = Ctx.getOperandInfo(Op1);
1438 [](
VPValue *
Op) {
return Op->getUnderlyingValue(); }))
1440 return Ctx.TTI.getArithmeticInstrCost(
1441 IsLogicalOr ? Instruction::Or : Instruction::And, ResultTy,
1442 Ctx.CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, Operands,
SI);
1446 if (!IsScalarCond && VF.
isVector())
1453 Pred = Cmp->getPredicate();
1455 return Ctx.TTI.getCmpSelInstrCost(
1456 Instruction::Select, VectorTy, CondTy, Pred, Ctx.CostKind,
1457 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None},
SI);
1486 "Should only generate a vector value or single scalar, not scalars "
1494 case Instruction::Select: {
1503 return Ctx.TTI.getCmpSelInstrCost(Instruction::Select, VecTy, CondTy, Pred,
1506 case Instruction::ExtractElement:
1516 return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
1521 return Ctx.TTI.getArithmeticReductionCost(
1528 return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ScalarTy,
1535 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
1541 return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ScalarTy,
1550 Cost += Ctx.TTI.getArithmeticInstrCost(
1551 Instruction::Xor, PredTy, Ctx.CostKind,
1552 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
1553 {TargetTransformInfo::OK_UniformConstantValue,
1554 TargetTransformInfo::OP_None});
1556 Cost += Ctx.TTI.getArithmeticInstrCost(Instruction::Sub, Ty, Ctx.CostKind);
1564 Intrinsic::experimental_vector_extract_last_active, ScalarTy,
1565 {VecTy, MaskTy, ScalarTy});
1566 return Ctx.TTI.getIntrinsicInstrCost(ICA, Ctx.CostKind);
1571 return Ctx.TTI.getShuffleCost(
1578 uint64_t Multiplier =
1585 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
1592 I32Ty, {Arg0Ty, I32Ty, I1Ty});
1593 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
1596 assert(VF.
isVector() &&
"Reverse operation must be vector type");
1605 VectorTy, Ctx.CostKind, {},
1611 return Ctx.TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
1612 VecTy, Ctx.CostKind, 0);
1622 return Ctx.TTI.getArithmeticInstrCost(Instruction::Xor, ValTy,
1628 return Ctx.TTI.getExtendedReductionCost(
1645 return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ValTy,
1655 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
1668 case Instruction::FCmp:
1669 case Instruction::ICmp:
1681 "unexpected VPInstruction witht underlying value");
1689 getOpcode() == Instruction::ExtractElement ||
1701 case Instruction::Load:
1702 case Instruction::PHI:
1714 Type *Ty =
Op->getScalarType();
1720 "types of operand 0 and new operand must match");
1726 "appended operand must match operand 0's scalar type");
1730 "appended operand must match operand 1's scalar type");
1735 constexpr unsigned NumInitialOperands = 3;
1737 "ExtractLastActive must have at least the initial 3 operands");
1738 bool IsMaskSlot = ((
getNumOperands() - NumInitialOperands) & 1u) == 1u;
1739 assert((IsMaskSlot ? Ty->isIntegerTy(1)
1741 "ExtractLastActive expects alternating data/mask operands "
1742 "matching operand 1's type and i1, respectively");
1747 "outside of construction");
1754 assert(!isAbstract() &&
"cannot execute this VPInstruction");
1757 "Set flags not supported for the provided opcode");
1759 "Opcode requires specific flags to be set");
1761 bool GenerateSingleScalar = State.VF.isScalar() || doesGenerateSingleScalar();
1762 Value *GeneratedValue = generate(State, GenerateSingleScalar);
1765 assert(GeneratedValue &&
"generate must produce a value");
1767 "scalar value but not only first lane defined");
1768 State.set(
this, GeneratedValue, GenerateSingleScalar);
1784 case Instruction::ExtractValue:
1785 case Instruction::GetElementPtr:
1786 case Instruction::ExtractElement:
1787 case Instruction::InsertElement:
1788 case Instruction::Freeze:
1789 case Instruction::FCmp:
1790 case Instruction::ICmp:
1791 case Instruction::Select:
1792 case Instruction::PHI:
1830 case Instruction::Call:
1845 case Instruction::ExtractElement:
1847 case Instruction::InsertElement:
1851 case Instruction::PHI:
1852 case Instruction::Load:
1888#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1896 O << Indent <<
"EMIT" << (
isSingleScalar() ?
"-SCALAR" :
"") <<
" ";
1908 O <<
"active lane mask";
1911 O <<
"wide active lane mask";
1914 O <<
"wide vector load";
1917 O <<
"wide vector store";
1920 O <<
"concat-vectors";
1923 O <<
"incoming-alias-mask";
1926 O <<
"EXPLICIT-VECTOR-LENGTH";
1929 O <<
"first-order splice";
1932 O <<
"branch-on-cond";
1935 O <<
"branch-on-two-conds";
1941 O <<
"branch-on-count";
1950 O <<
"exiting-iv-value";
1956 O <<
"extract-lane";
1959 O <<
"extract-last-lane";
1962 O <<
"extract-last-part";
1965 O <<
"extract-penultimate-element";
1968 O <<
"extract-vector-for-part";
1971 O <<
"compute-reduction-result";
1989 O <<
"first-active-lane";
1992 O <<
"last-active-lane";
1995 O <<
"reduction-start-vector";
1998 O <<
"resume-for-epilogue";
2007 O <<
"extract-last-active";
2010 O <<
"num-active-lanes";
2013 O <<
"wide-iv-step";
2019 case Instruction::Call: {
2034 O, [&O, &
SlotTracker,
this](
const auto &IndexedOp) {
2035 auto [Idx,
Op] = IndexedOp;
2047 case Instruction::Load:
2069 const Twine &Name) {
2072 : Phi.getNumIncoming();
2073 Value *FirstInc = State.get(Phi.getIncomingValue(0), IsScalar);
2074 PHINode *NewPhi = State.Builder.CreatePHI(FirstInc->
getType(), 2, Name);
2076 State.CFG.VPBB2IRBB.at(Phi.getIncomingBlock(0)));
2077 for (
unsigned Idx = 1; Idx != NumIncoming; ++Idx)
2078 NewPhi->
addIncoming(State.get(Phi.getIncomingValue(Idx), IsScalar),
2079 State.CFG.VPBB2IRBB.at(Phi.getIncomingBlock(Idx)));
2080 State.set(R, NewPhi, IsScalar);
2087#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2090 O << Indent <<
"EMIT" << (
isSingleScalar() ?
"-SCALAR" :
"") <<
" ";
2106 "PHINodes must be handled by VPIRPhi");
2109 State.Builder.SetInsertPoint(std::next(I.getIterator()));
2119#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2122 O << Indent <<
"IR " << I;
2134 auto *PredVPBB = Pred->getExitingBasicBlock();
2135 BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB];
2142 if (Phi->getBasicBlockIndex(PredBB) == -1)
2143 Phi->addIncoming(V, PredBB);
2145 Phi->setIncomingValueForBlock(PredBB, V);
2150 State.Builder.SetInsertPoint(std::next(Phi->getIterator()));
2155 assert(R->getNumOperands() == R->getParent()->getNumPredecessors() &&
2156 "Number of phi operands must match number of predecessors");
2157 unsigned Position = R->getParent()->getIndexForPredecessor(IncomingBlock);
2158 R->removeOperand(Position);
2170 R->setOperand(R->getParent()->getIndexForPredecessor(VPBB), V);
2173#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2180 std::get<1>(
Op)->printAsOperand(O);
2186#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2192 O <<
" (extra operand" << (
getNumOperands() > 1 ?
"s" :
"") <<
": ";
2197 std::get<1>(
Op)->printAsOperand(O);
2205 if (Metadata.empty())
2209 unsigned ExecFreqKind = getMDKindID(ExecutionFrequencyMDName);
2210 unsigned EstProfKind = getMDKindID(EstimatedProfileMDName);
2211 for (
const auto &[Kind,
Node] : Metadata)
2212 if (Kind != ExecFreqKind && Kind != EstProfKind)
2213 I.setMetadata(Kind,
Node);
2218 assert(
Node->getNumOperands() <= 2 &&
"unexpected frequency node shape");
2222 "frequency cannot exceed the one of an always executing block");
2227 std::optional<VPExecutionFrequency> Freq,
LLVMContext &Ctx) {
2233 if (Freq->IsEstimated)
2238std::optional<VPExecutionFrequency>
2240 if (
MDNode *
Node = getInternalMetadata(ExecutionFrequencyMDName))
2242 return std::nullopt;
2246 if (Metadata.empty())
2248 unsigned ID = getMDKindID(ExecutionFrequencyMDName);
2249 erase_if(Metadata, [ID](
const auto &
P) {
return P.first == ID; });
2257#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2260 if (Metadata.empty() || !M)
2266 auto [Kind,
Node] = KindNodePair;
2268 "Unexpected unnamed metadata kind");
2269 O <<
"!" << MDNames[Kind] <<
" ";
2273 bool IsEstimatedProfile = MDNames[Kind] == EstimatedProfileMDName;
2274 if ((Kind == LLVMContext::MD_prof || IsEstimatedProfile) &&
2276 if (IsEstimatedProfile)
2281 }
else if (MDNames[Kind] == ExecutionFrequencyMDName) {
2290 Percent.toString(PercentStr, 4);
2291 O << Freq.getFrequency() <<
" (" << PercentStr <<
"%"
2292 << (IsEstimated ?
", estimated" :
"") <<
")";
2303 assert(
Call.getOpcode() == Instruction::Call &&
"expected call");
2305 assert(NumArgs <= WideTy->getNumParams() &&
2306 "widened call must take all scalar arguments");
2314 Attribute::Alignment, Attribute::NonNull, Attribute::NoFPClass,
2320 AttributeMask Incompatible = AttributeFuncs::typeIncompatible(Ty, AS);
2321 AttrBuilder AB(Ctx);
2323 if (AS.hasAttribute(
K) && !Incompatible.
contains(
K))
2324 AB.addAttribute(AS.getAttribute(
K));
2336 bool IsMaskedVariant = WideTy->
getNumParams() > NumArgs;
2338 for (
unsigned I = 0;
I != NumArgs; ++
I) {
2340 bool IsWidenedArg = WideArgTy !=
Call.getOperand(
I)->getScalarType();
2341 if (
Call.isMasked() && (!IsMaskedVariant || !IsWidenedArg))
2343 ArgAttrs[
I] =
Filter(WideArgTy, ScalarAttrs.getParamAttrs(
I));
2346 CallAttrs = AttributeList::get(Ctx,
AttributeSet(), RetAttrs, ArgAttrs);
2355#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2376 assert(State.VF.isVector() &&
"not widening");
2377 assert(Variant !=
nullptr &&
"Can't create vector function.");
2388 Arg = State.get(
I.value(),
VPLane(0));
2391 Args.push_back(Arg);
2394 CallInst *V = State.Builder.CreateCall(Variant, Args);
2398 V->setCallingConv(Variant->getCallingConv());
2400 if (!V->getType()->isVoidTy())
2407 "Variant return type must match VF");
2413 return Ctx.TTI.getCallInstrCost(
nullptr, Variant->getReturnType(),
2414 Variant->getFunctionType()->params(),
2420 assert(Variant &&
"Variant not set");
2423 auto [Idx, V] = Arg;
2430#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2433 O << Indent <<
"WIDEN-CALL ";
2446 O <<
"@" << CalledFn->
getName() <<
"(";
2449 auto [Idx,
Op] = IndexedOp;
2455 O <<
" (using library function";
2456 if (Variant->hasName())
2457 O <<
": " << Variant->getName();
2463 assert(State.VF.isVector() &&
"not widening");
2471 for (
auto [Idx, Ty] :
enumerate(ContainedTys)) {
2484 Arg = State.get(
I.value(),
VPLane(0));
2490 Args.push_back(Arg);
2494 Module *M = State.Builder.getModule();
2498 "Can't retrieve vector intrinsic or vector-predication intrinsics.");
2500 CallInst *V = State.Builder.CreateCall(VectorF, Args);
2510 if (!V->getType()->isVoidTy())
2517 Type *ScalarRetTy = R.getScalarType();
2521 if (ID == Intrinsic::experimental_vp_reverse && ScalarRetTy->
isIntegerTy(1))
2531 auto *V =
Op->getUnderlyingValue();
2534 Arguments.push_back(UI->getArgOperand(Idx));
2557 if (
is_contained({Intrinsic::umin, Intrinsic::umax, Intrinsic::smin,
2559 WidenIntrinsic->getVectorIntrinsicID())) {
2568 ID, RetTy,
Arguments, ParamTys, R.getFastMathFlagsOrNone(),
2571 return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, Ctx.CostKind);
2592#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2595 O << Indent <<
"WIDEN-INTRINSIC ";
2616 State.set(
this, MemI);
2621 assert(MaskPos &&
"Expected a memory intrinsic with a valid mask position");
2627 assert(PtrPos &&
"Expected a memory intrinsic with a valid pointer position");
2644 return Ctx.TTI.getMemIntrinsicInstrCost(
2666 Value *Mask =
nullptr;
2668 Mask = State.get(VPMask);
2671 Builder.CreateVectorSplat(VTy->
getElementCount(), Builder.getInt1(1));
2675 if (mustNegateIncrement())
2676 IncAmt = Builder.CreateNeg(IncAmt);
2678 Instruction *HistogramInst = State.Builder.CreateIntrinsicWithoutFolding(
2679 getHistogramIntrinsicID(), {VTy, IncAmt->
getType()},
2684Intrinsic::ID VPHistogramRecipe::getHistogramIntrinsicID()
const {
2685 switch (UpdateKind) {
2688 return Intrinsic::experimental_vector_histogram_add;
2690 return Intrinsic::experimental_vector_histogram_uadd_sat;
2692 return Intrinsic::experimental_vector_histogram_umax;
2694 return Intrinsic::experimental_vector_histogram_umin;
2719 Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
2726 {PtrTy, IncTy, MaskTy});
2730 return Ctx.TTI.getIntrinsicInstrCost(ICA, Ctx.CostKind) + MulCost;
2733#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2736 O << Indent <<
"WIDEN-HISTOGRAM buckets: ";
2739 switch (UpdateKind) {
2747 O <<
", saturated inc: ";
2765VPIRFlags::FastMathFlagsTy::FastMathFlagsTy(
const FastMathFlags &FMF) {
2777 case Instruction::Add:
2778 case Instruction::Sub:
2779 case Instruction::Mul:
2780 case Instruction::Shl:
2783 case Instruction::Trunc:
2785 case Instruction::Or:
2787 case Instruction::AShr:
2788 case Instruction::LShr:
2789 case Instruction::UDiv:
2790 case Instruction::SDiv:
2791 return ExactFlagsTy(
false);
2792 case Instruction::GetElementPtr:
2796 case Instruction::ZExt:
2797 case Instruction::UIToFP:
2799 case Instruction::FAdd:
2800 case Instruction::FSub:
2801 case Instruction::FMul:
2802 case Instruction::FDiv:
2803 case Instruction::FRem:
2804 case Instruction::FNeg:
2805 case Instruction::FPExt:
2806 case Instruction::FPTrunc:
2808 case Instruction::Select:
2809 case Instruction::PHI:
2810 case Instruction::Call:
2816 case Instruction::ICmp:
2817 case Instruction::FCmp:
2828 case OperationType::OverflowingBinOp:
2829 return Opcode == Instruction::Add || Opcode == Instruction::Sub ||
2830 Opcode == Instruction::Mul || Opcode == Instruction::Shl ||
2831 Opcode == VPInstruction::VPInstruction::CanonicalIVIncrementForPart;
2832 case OperationType::Trunc:
2833 return Opcode == Instruction::Trunc;
2834 case OperationType::DisjointOp:
2835 return Opcode == Instruction::Or;
2836 case OperationType::PossiblyExactOp:
2837 return Opcode == Instruction::AShr || Opcode == Instruction::LShr ||
2838 Opcode == Instruction::UDiv || Opcode == Instruction::SDiv;
2839 case OperationType::GEPOp:
2840 return Opcode == Instruction::GetElementPtr ||
2843 case OperationType::FPMathOp:
2844 return Opcode == Instruction::Call || Opcode == Instruction::FAdd ||
2845 Opcode == Instruction::FMul || Opcode == Instruction::FSub ||
2846 Opcode == Instruction::FNeg || Opcode == Instruction::FDiv ||
2847 Opcode == Instruction::FRem || Opcode == Instruction::FPExt ||
2848 Opcode == Instruction::FPTrunc || Opcode == Instruction::PHI ||
2849 Opcode == Instruction::Select || Opcode == Instruction::SIToFP ||
2850 Opcode == Instruction::UIToFP ||
2853 case OperationType::FCmp:
2854 return Opcode == Instruction::FCmp;
2855 case OperationType::NonNegOp:
2856 return Opcode == Instruction::ZExt || Opcode == Instruction::UIToFP;
2857 case OperationType::Cmp:
2858 return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;
2859 case OperationType::ReductionOp:
2861 case OperationType::Other:
2868 Type *ResultTy)
const {
2870 if (Opcode == Instruction::ICmp)
2871 return OpType == OperationType::Cmp;
2872 if (Opcode == Instruction::FCmp)
2873 return OpType == OperationType::FCmp;
2875 return OpType == OperationType::ReductionOp;
2878 return Required == OperationType::Other || Required == OpType;
2882#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2895 OS <<
"add-chain-with-subs";
2925 OS <<
"fadd-chain-with-subs";
2952 OS <<
"fminimumnum";
2955 OS <<
"fmaximumnum";
2974 case OperationType::Cmp:
2977 case OperationType::FCmp:
2981 case OperationType::DisjointOp:
2985 case OperationType::PossiblyExactOp:
2989 case OperationType::OverflowingBinOp:
2995 case OperationType::Trunc:
3001 case OperationType::FPMathOp:
3004 case OperationType::GEPOp: {
3006 if (Flags.isInBounds())
3008 else if (Flags.hasNoUnsignedSignedWrap())
3010 if (Flags.hasNoUnsignedWrap())
3014 case OperationType::NonNegOp:
3018 case OperationType::ReductionOp: {
3029 case OperationType::Other:
3037 auto &Builder = State.Builder;
3039 case Instruction::Call:
3040 case Instruction::UncondBr:
3041 case Instruction::CondBr:
3042 case Instruction::PHI:
3043 case Instruction::GetElementPtr:
3045 case Instruction::UDiv:
3046 case Instruction::SDiv:
3047 case Instruction::SRem:
3048 case Instruction::URem:
3049 case Instruction::Add:
3050 case Instruction::FAdd:
3051 case Instruction::Sub:
3052 case Instruction::FSub:
3053 case Instruction::FNeg:
3054 case Instruction::Mul:
3055 case Instruction::FMul:
3056 case Instruction::FDiv:
3057 case Instruction::FRem:
3058 case Instruction::Shl:
3059 case Instruction::LShr:
3060 case Instruction::AShr:
3061 case Instruction::And:
3062 case Instruction::Or:
3063 case Instruction::Xor: {
3067 Ops.push_back(State.get(VPOp));
3069 Value *V = Builder.CreateNAryOp(Opcode,
Ops);
3080 case Instruction::ExtractValue: {
3084 State.set(
this, Extract);
3087 case Instruction::Freeze: {
3089 Value *Freeze = Builder.CreateFreeze(
Op);
3090 State.set(
this, Freeze);
3093 case Instruction::ICmp:
3094 case Instruction::FCmp: {
3096 bool FCmp = Opcode == Instruction::FCmp;
3112 case Instruction::Select: {
3117 Value *Sel = State.Builder.CreateSelect(
Cond, Op0, Op1);
3118 State.set(
this, Sel);
3137 State.get(
this)->getType() &&
3138 "inferred type and type from generated instructions do not match");
3145 case Instruction::UDiv:
3146 case Instruction::SDiv:
3147 case Instruction::SRem:
3148 case Instruction::URem:
3153 case Instruction::FNeg:
3154 case Instruction::Add:
3155 case Instruction::FAdd:
3156 case Instruction::Sub:
3157 case Instruction::FSub:
3158 case Instruction::Mul:
3159 case Instruction::FMul:
3160 case Instruction::FDiv:
3161 case Instruction::FRem:
3162 case Instruction::Shl:
3163 case Instruction::LShr:
3164 case Instruction::AShr:
3165 case Instruction::And:
3166 case Instruction::Or:
3167 case Instruction::Xor:
3168 case Instruction::Freeze:
3169 case Instruction::ExtractValue:
3170 case Instruction::ICmp:
3171 case Instruction::FCmp:
3172 case Instruction::Select:
3179#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3182 O << Indent <<
"WIDEN ";
3191 auto &Builder = State.Builder;
3193 assert(State.VF.isVector() &&
"Not vectorizing?");
3198 State.set(
this, Cast);
3210#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3213 O << Indent <<
"WIDEN-CAST ";
3224 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
3227#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3232 O <<
" = WIDEN-INDUCTION";
3260 : ID.getInductionOpcode();
3261 assert(IncOpc != Instruction::BinaryOpsEnd &&
3262 "induction must have a valid increment opcode");
3263 return Cost + Ctx.TTI.getArithmeticInstrCost(IncOpc,
toVectorTy(StepTy, VF),
3290 bool NeedsMul =
true, NeedsAdd =
true, NeedsShl =
false;
3294 NeedsAdd = !StartC->isZero();
3305 else if (StepC->getAPInt().isAllOnes()) {
3312 }
else if (StepC->getAPInt().isPowerOf2()) {
3324 if ((NeedsAdd || NeedsMul || NeedsShl) && StepTySize != IndexTySize) {
3326 StepTySize < IndexTySize ? Instruction::Trunc : Instruction::ZExt;
3327 Cost += Ctx.TTI.getCastInstrCost(
3332 Cost += Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, StepTy,
3335 Cost += Ctx.TTI.getArithmeticInstrCost(
3336 Instruction::Shl, StepTy, Ctx.CostKind,
3337 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
3338 {TargetTransformInfo::OK_UniformConstantValue,
3339 TargetTransformInfo::OP_None});
3341 Cost += Ctx.TTI.getArithmeticInstrCost(Instruction::Add, StepTy,
3357 Ctx.TTI.getCastInstrCost(Instruction::UIToFP, StepTy, IndexTy,
3363 bool NeedsMul = !StepC || !StepC->
isOne();
3375 Cost += Ctx.TTI.getArithmeticInstrCost(Instruction::FMul, StepTy,
3378 Cost += Ctx.TTI.getArithmeticInstrCost(
getFPBinOp()->getOpcode(), StepTy,
3387#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3392 O <<
" = DERIVED-IV";
3410 "VPScalarIVStepsRecipe is only created for integer and FP inductions");
3433 "FP scalar steps for all lanes are only created for fixed VFs");
3434 Cost = Ctx.TTI.getArithmeticInstrCost(InductionOpcode, BaseIVTy,
3450 Cost = Ctx.TTI.getArithmeticInstrCost(Instruction::Add, BaseIVTy,
3458 Cost /= Ctx.getCostDivisor(
3459 Region->getEntryBranchOnMask()->getExecutionFrequency());
3477 assert(BaseIVTy == Step->
getType() &&
"Types of BaseIV and Step must match!");
3484 AddOp = Instruction::Add;
3485 MulOp = Instruction::Mul;
3487 AddOp = InductionOpcode;
3488 MulOp = Instruction::FMul;
3494 "must have been replicated by VF");
3497 auto *
Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);
3498 auto *
Add = Builder.CreateBinOp(AddOp, BaseIV,
Mul);
3502#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3507 O <<
" = SCALAR-STEPS ";
3518 assert(State.VF.isVector() &&
"not widening");
3528#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3531 O << Indent <<
"WIDEN-GEP ";
3533 O <<
" = getelementptr";
3556 Builder.createOverflowingOp(Instruction::Mul, {VFMinusOne, Stride});
3563 Builder.createOverflowingOp(Instruction::Mul, {PartxStride, VF}));
3568 auto &Builder = State.Builder;
3574 State.set(
this, ResultPtr,
true);
3577#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3582 O <<
" = vector-end-pointer";
3592 "Expected prior simplification of recipe without VFxPart");
3594 auto &Builder = State.Builder;
3599 Value *Stride = Builder.CreateZExtOrTrunc(State.get(
getStride(),
true),
3605 State.set(
this, ResultPtr,
true);
3608#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3613 O <<
" = vector-pointer";
3637 Cost += Ctx.TTI.getCmpSelInstrCost(Instruction::Select, ResultTy, CmpTy,
3638 Pred, Ctx.CostKind);
3643#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3646 O << Indent <<
"BLEND ";
3671 "In-loop AnyOf reductions aren't currently supported");
3677 Value *NewCond = State.get(
Cond, State.VF.isScalar());
3683 if (State.VF.isVector())
3684 Start = State.Builder.CreateVectorSplat(VecTy->
getElementCount(), Start);
3686 Value *
Select = State.Builder.CreateSelect(NewCond, NewVecOp, Start);
3693 if (State.VF.isVector())
3697 NewRed = State.Builder.CreateBinOp(
3699 PrevInChain, NewVecOp);
3700 PrevInChain = NewRed;
3701 NextInChain = NewRed;
3704 "Unexpected partial reduction kind");
3706 NewRed = State.Builder.CreateIntrinsic(
3709 : Intrinsic::vector_partial_reduce_fadd,
3710 {PrevInChain, NewVecOp}, State.Builder.getFastMathFlags(),
3712 PrevInChain = NewRed;
3713 NextInChain = NewRed;
3716 "The reduction must either be ordered, partial or in-loop");
3720 NextInChain =
createMinMaxOp(State.Builder, Kind, NewRed, PrevInChain);
3722 NextInChain = State.Builder.CreateBinOp(
3724 PrevInChain, NewRed);
3731 assert(State.VF.isVector() &&
3732 "Shouldn't generate VPReductionEVLRecipe with scalar VF");
3733 auto &Builder = State.Builder;
3746 Mask = State.get(CondOp);
3748 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
3762 Value *NewVecOp = State.Builder.CreateIntrinsic(
3763 VecTy, Intrinsic::vp_merge, {Mask, VecOp, Identity, EVL});
3765 "Unexpected partial reduction kind");
3766 NewRed = State.Builder.CreateIntrinsic(
3769 : Intrinsic::vector_partial_reduce_fadd,
3770 {Prev, NewVecOp}, State.Builder.getFastMathFlags(),
"partial.reduce");
3778 NewRed = Builder.CreateBinOp(
3792 std::optional<FastMathFlags> OptionalFMF =
3801 CondCost = Ctx.TTI.getCmpSelInstrCost(Instruction::Select, VectorTy,
3802 CondTy, Pred, Ctx.CostKind);
3804 return CondCost + Ctx.TTI.getPartialReductionCost(
3805 Opcode, ElementTy,
nullptr, ElementTy, VF,
3813 Ctx.Opts.force_target_instruction_cost) &&
3814 "Any-of reduction not implemented in VPlan-based cost model currently.");
3820 return Ctx.TTI.getMinMaxReductionCost(Id, VectorTy,
FMFs, Ctx.CostKind);
3825 return Ctx.TTI.getArithmeticReductionCost(Opcode, VectorTy, OptionalFMF,
3830 ExpressionTypes ExpressionType,
3834 if (ExpressionType == ExpressionTypes::FoldedVPMerge)
3835 return ExpressionRecipes.back()->getScalarType();
3837 ExpressionRecipes.back())
3841 ExpressionRecipes(ExpressionRecipes), ExpressionType(ExpressionType) {
3842 assert(!ExpressionRecipes.empty() &&
"Nothing to combine?");
3846 "expression cannot contain recipes with side-effects");
3850 for (
auto *R : ExpressionRecipes)
3851 ExpressionRecipesAsSetOfUsers.
insert(R);
3857 if (R != ExpressionRecipes.back() &&
3858 any_of(
R->users(), [&ExpressionRecipesAsSetOfUsers](
VPUser *U) {
3859 return !ExpressionRecipesAsSetOfUsers.contains(U);
3864 R->replaceUsesWithIf(CopyForExtUsers,
3865 [&ExpressionRecipesAsSetOfUsers](
VPUser *U) {
3866 return !ExpressionRecipesAsSetOfUsers.contains(U);
3871 R->removeFromParent();
3878 for (
auto *R : ExpressionRecipes) {
3879 for (
const auto &[Idx,
Op] :
enumerate(
R->operands())) {
3880 auto *
Def =
Op->getDefiningRecipe();
3881 if (Def && ExpressionRecipesAsSetOfUsers.contains(Def))
3890 for (
auto *R : ExpressionRecipes)
3891 for (
auto const &[LiveIn, Tmp] :
zip(operands(), LiveInPlaceholders))
3892 R->replaceUsesOfWith(LiveIn, Tmp);
3896 for (
auto *R : ExpressionRecipes)
3899 if (!R->getParent())
3900 R->insertBefore(
this);
3903 LiveInPlaceholders[Idx]->replaceAllUsesWith(
Op);
3907 ExpressionRecipes.clear();
3908 return DecomposedRecipes;
3914 if (ExpressionType == ExpressionTypes::FoldedVPMerge &&
3917 "Expected an explicit vector length recipe");
3927 if (ExpressionType == ExpressionTypes::FoldedVPMerge) {
3929 for (
auto *R :
drop_end(ExpressionRecipes))
3930 Cost += R->cost(VF, Ctx);
3939 switch (ExpressionType) {
3940 case ExpressionTypes::NegatedExtendedReduction:
3941 assert((Opcode == Instruction::Add || Opcode == Instruction::FAdd) &&
3942 "Unexpected opcode");
3943 Opcode = Opcode == Instruction::Add ? Instruction::Sub : Instruction::FSub;
3945 case ExpressionTypes::ExtendedReduction: {
3949 if (RedR->isPartialReduction())
3950 return Ctx.TTI.getPartialReductionCost(
3955 ? std::optional{RedR->getFastMathFlagsOrNone()}
3959 return Ctx.TTI.getExtendedReductionCost(
3960 Opcode, ExtR->getOpcode() == Instruction::ZExt, RedTy, SrcVecTy,
3961 std::nullopt, Ctx.CostKind);
3965 case ExpressionTypes::MulAccReduction:
3966 return Ctx.TTI.getMulAccReductionCost(
false, Opcode, RedTy, SrcVecTy,
3969 case ExpressionTypes::ExtNegatedMulAccReduction:
3971 case Instruction::Add:
3972 Opcode = Instruction::Sub;
3974 case Instruction::FAdd:
3975 Opcode = Instruction::FSub;
3981 case ExpressionTypes::ExtMulAccReduction: {
3983 if (RedR->isPartialReduction()) {
3987 return Ctx.TTI.getPartialReductionCost(
3991 Ext0R->getOpcode()),
3993 Ext1R->getOpcode()),
3994 Mul->getOpcode(), Ctx.CostKind,
3996 ? std::optional{RedR->getFastMathFlagsOrNone()}
3999 assert(Opcode != Instruction::FSub &&
"Only integer types are supported");
4000 return Ctx.TTI.getMulAccReductionCost(
4003 Opcode, RedTy, SrcVecTy, Ctx.CostKind);
4005 case ExpressionTypes::FoldedVPMerge:
4013 return R->mayReadFromMemory() || R->mayWriteToMemory();
4021 "expression cannot contain recipes with side-effects");
4027 return RR && !RR->isPartialReduction();
4030#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4034 O << Indent <<
"EXPRESSION ";
4037 if (ExpressionType == ExpressionTypes::FoldedVPMerge) {
4043 auto PrintInLoopOperands = [&]() {
4053 O << WidenIntrinsic->getIntrinsicName();
4054 WidenIntrinsic->printFlags(O);
4055 PrintInLoopOperands();
4058 Widen->printFlags(O);
4059 PrintInLoopOperands();
4079 getNumOperands() - (Red->isConditional() ? 2 : 1) - (EVL ? 1 : 0));
4080 auto PrintEVLAndMask = [&]() {
4085 if (Red->isConditional()) {
4091 switch (ExpressionType) {
4092 case ExpressionTypes::NegatedExtendedReduction:
4093 case ExpressionTypes::ExtendedReduction: {
4094 bool Negated = ExpressionType == ExpressionTypes::NegatedExtendedReduction;
4096 O <<
" + " << (Red->isPartialReduction() ?
"partial." :
"") <<
"reduce.";
4099 O << (Opcode == Instruction::Add ?
"sub (0, " :
"fneg(");
4107 << *Ext0->getScalarType();
4112 case ExpressionTypes::ExtNegatedMulAccReduction: {
4114 O <<
" + " << (Red->isPartialReduction() ?
"partial." :
"") <<
"reduce.";
4124 << *Ext0->getScalarType() <<
"), (";
4128 << *Ext1->getScalarType() <<
")";
4133 case ExpressionTypes::MulAccReduction:
4134 case ExpressionTypes::ExtMulAccReduction: {
4136 O <<
" + " << (Red->isPartialReduction() ?
"partial." :
"") <<
"reduce.";
4141 bool IsExtended = ExpressionType == ExpressionTypes::ExtMulAccReduction;
4143 : ExpressionRecipes[0]);
4151 << *Ext0->getScalarType() <<
"), (";
4159 << *Ext1->getScalarType() <<
")";
4173 O << Indent <<
"PARTIAL-REDUCE ";
4175 O << Indent <<
"REDUCE ";
4195 O << Indent <<
"PARTIAL-REDUCE ";
4197 O << Indent <<
"REDUCE ";
4221 "VPReplicateRecipes must be unrolled before ::execute");
4226 Cloned->
setName(Instr->getName() +
".cloned");
4230 if (ResultTy != Cloned->
getType())
4249 State.Builder.Insert(Cloned);
4251 State.set(
this, Cloned,
true);
4255 State.AC->registerAssumption(
II);
4278 Ctx.SkipCostComputation.insert(UI);
4284 case Instruction::Alloca:
4287 return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul,
4289 case Instruction::GetElementPtr:
4295 case Instruction::Call: {
4301 case Instruction::FNeg:
4302 case Instruction::Add:
4303 case Instruction::Sub:
4304 case Instruction::FAdd:
4305 case Instruction::FSub:
4306 case Instruction::Mul:
4307 case Instruction::FMul:
4308 case Instruction::FDiv:
4309 case Instruction::FRem:
4310 case Instruction::Shl:
4311 case Instruction::LShr:
4312 case Instruction::AShr:
4313 case Instruction::And:
4314 case Instruction::Or:
4315 case Instruction::Xor:
4316 case Instruction::ICmp:
4317 case Instruction::FCmp:
4318 case Instruction::Freeze:
4322 case Instruction::SDiv:
4323 case Instruction::UDiv:
4324 case Instruction::SRem:
4325 case Instruction::URem: {
4338 return Ctx.skipCostComputation(
4340 PredR->getOperand(0)->getUnderlyingValue()),
4356 Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
4360 ScalarCost /= Ctx.getCostDivisor(
4364 case Instruction::Load:
4365 case Instruction::Store: {
4366 bool IsLoad = UI->
getOpcode() == Instruction::Load;
4383 bool PreferVectorizedAddressing = Ctx.TTI.prefersVectorizedAddressing();
4384 bool UsedByLoadStoreAddress =
4387 UI->
getOpcode(), ValTy, Alignment, AS, Ctx.CostKind, OpInfo,
4388 UsedByLoadStoreAddress ? UI :
nullptr);
4393 Ctx.TTI.getAddressComputationCost(
4394 PtrTy, UsedByLoadStoreAddress ?
nullptr : Ctx.PSE.getSE(), PtrSCEV,
4405 if (!UsedByLoadStoreAddress) {
4406 bool EfficientVectorLoadStore =
4407 Ctx.TTI.supportsEfficientVectorElementLoadStore();
4408 if (!(IsLoad && !PreferVectorizedAddressing) &&
4409 !(!IsLoad && EfficientVectorLoadStore))
4412 if (!EfficientVectorLoadStore)
4417 IsLoad ? TTI::VectorInstrContext::Load : TTI::VectorInstrContext::Store;
4420 Ctx.getScalarizationOverhead(ResultTy, OpsToScalarize, VF, VIC,
true);
4425 Cost /= Ctx.getCostDivisor(
4428 if (Ctx.useEmulatedMaskMemRefHack(
this, VF)) {
4436 case Instruction::BitCast:
4437 case Instruction::SExt:
4438 case Instruction::ZExt:
4439 case Instruction::FPToUI:
4440 case Instruction::FPToSI:
4441 case Instruction::FPExt:
4442 case Instruction::PtrToInt:
4443 case Instruction::PtrToAddr:
4444 case Instruction::IntToPtr:
4445 case Instruction::SIToFP:
4446 case Instruction::UIToFP:
4447 case Instruction::Trunc:
4448 case Instruction::FPTrunc:
4449 case Instruction::Select:
4450 case Instruction::AddrSpaceCast: {
4455 case Instruction::ExtractValue:
4456 return Ctx.TTI.getInsertExtractValueCost(
getOpcode(), Ctx.CostKind);
4459 return Ctx.getLegacyCost(UI, VF);
4466 ArgOps, [&](
const VPValue *
Op) {
return Op->getScalarType(); });
4469 auto GetIntrinsicCost = [&] {
4472 return Ctx.TTI.getIntrinsicInstrCost(
4477 assert(GetIntrinsicCost() == 0 &&
"scalarizing intrinsic should be free");
4482 Ctx.TTI.getCallInstrCost(CalledFn, ResultTy, Tys, Ctx.CostKind);
4483 if (IsSingleScalar) {
4484 ScalarCallCost = std::min(ScalarCallCost, GetIntrinsicCost());
4485 return ScalarCallCost;
4493 Ctx.getScalarizationOverhead(ResultTy, ArgOps, VF);
4496#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4499 O << Indent << (IsSingleScalar ?
"CLONE " :
"REPLICATE ");
4508 O <<
"@" << CB->getCalledFunction()->getName() <<
"(";
4531 llvm_unreachable(
"recipe must be removed when dissolving replicate region");
4540 return Ctx.TTI.getScalarizationOverhead(
4542 false,
true, Ctx.CostKind) +
4544 Ctx.TTI.getCFInstrCost(Instruction::CondBr, Ctx.CostKind);
4548 llvm_unreachable(
"recipe must be removed when dissolving replicate region");
4551#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4554 O << Indent <<
"PHI-PREDICATED-INSTRUCTION ";
4578 : R->getOperand(1)->getScalarType();
4582 unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
4599 : Intrinsic::vp_scatter;
4600 return Ctx.TTI.getAddressComputationCost(PtrTy,
nullptr,
nullptr,
4602 Ctx.TTI.getMemIntrinsicInstrCost(
4611 : Intrinsic::masked_store;
4612 Cost += Ctx.TTI.getMemIntrinsicInstrCost(
4617 : R->getOperand(1));
4618 Cost += Ctx.TTI.getMemoryOpCost(Opcode, Ty,
Alignment, AS, Ctx.CostKind,
4629 auto &Builder = State.Builder;
4630 Value *Mask =
nullptr;
4632 Mask = State.get(VPMask);
4637 NewLI = Builder.CreateMaskedGather(DataTy, Addr,
Alignment, Mask,
nullptr,
4638 "wide.masked.gather");
4641 Builder.CreateMaskedLoad(DataTy, Addr,
Alignment, Mask,
4644 NewLI = Builder.CreateAlignedLoad(DataTy, Addr,
Alignment,
"wide.load");
4647 State.set(
this, NewLI);
4650#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4653 O << Indent <<
"WIDEN ";
4666 auto &Builder = State.Builder;
4670 Value *Mask =
nullptr;
4672 Mask = State.get(VPMask);
4674 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
4677 NewLI = Builder.CreateIntrinsicWithoutFolding(DataTy, Intrinsic::vp_gather,
4678 {Addr, Mask, EVL},
nullptr,
4679 "wide.masked.gather");
4681 NewLI = Builder.CreateIntrinsicWithoutFolding(
4682 DataTy, Intrinsic::vp_load, {Addr, Mask, EVL},
nullptr,
"vp.op.load");
4687 State.set(
this, NewLI);
4703 return Ctx.TTI.getMemIntrinsicInstrCost(
4708#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4711 O << Indent <<
"WIDEN ";
4723 auto &Builder = State.Builder;
4725 Value *Mask =
nullptr;
4727 Mask = State.get(VPMask);
4729 Value *StoredVal = State.get(StoredVPValue);
4733 NewSI = Builder.CreateMaskedScatter(StoredVal, Addr,
Alignment, Mask);
4735 NewSI = Builder.CreateMaskedStore(StoredVal, Addr,
Alignment, Mask);
4737 NewSI = Builder.CreateAlignedStore(StoredVal, Addr,
Alignment);
4741#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4744 O << Indent <<
"WIDEN store ";
4754 auto &Builder = State.Builder;
4757 Value *StoredVal = State.get(StoredValue);
4759 Value *Mask =
nullptr;
4761 Mask = State.get(VPMask);
4763 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
4766 if (CreateScatter) {
4767 NewSI = Builder.CreateIntrinsicWithoutFolding(
4769 {StoredVal, Addr, Mask, EVL});
4771 NewSI = Builder.CreateIntrinsicWithoutFolding(
4773 {StoredVal, Addr, Mask, EVL});
4793 return Ctx.TTI.getMemIntrinsicInstrCost(
4798#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4801 O << Indent <<
"WIDEN vp.store ";
4810 auto VF = DstVTy->getElementCount();
4812 assert(VF == SrcVecTy->getElementCount() &&
"Vector dimensions do not match");
4813 Type *SrcElemTy = SrcVecTy->getElementType();
4814 Type *DstElemTy = DstVTy->getElementType();
4815 assert((
DL.getTypeSizeInBits(SrcElemTy) ==
DL.getTypeSizeInBits(DstElemTy)) &&
4816 "Vector elements must have same size");
4820 return Builder.CreateBitOrPointerCast(
V, DstVTy);
4827 "Only one type should be a pointer type");
4829 "Only one type should be a floating point type");
4833 Value *CastVal = Builder.CreateBitOrPointerCast(
V, VecIntTy);
4834 return Builder.CreateBitOrPointerCast(CastVal, DstVTy);
4840 const Twine &Name) {
4841 unsigned Factor = Vals.
size();
4842 assert(Factor > 1 &&
"Tried to interleave invalid number of vectors");
4846 for (
Value *Val : Vals)
4847 assert(Val->getType() == VecTy &&
"Tried to interleave mismatched types");
4852 if (VecTy->isScalableTy()) {
4853 assert(Factor <= 8 &&
"Unsupported interleave factor for scalable vectors");
4854 return Builder.CreateVectorInterleave(Vals, Name);
4861 const unsigned NumElts = VecTy->getElementCount().getFixedValue();
4862 return Builder.CreateShuffleVector(
4896 "Masking gaps for scalable vectors is not yet supported.");
4902 unsigned InterleaveFactor = Group->
getFactor();
4909 auto CreateGroupMask = [&BlockInMask, &State,
4910 &InterleaveFactor](
Value *MaskForGaps) ->
Value * {
4911 if (State.VF.isScalable()) {
4912 assert(!MaskForGaps &&
"Interleaved groups with gaps are not supported.");
4913 assert(InterleaveFactor <= 8 &&
4914 "Unsupported deinterleave factor for scalable vectors");
4915 auto *ResBlockInMask = State.get(BlockInMask);
4923 Value *ResBlockInMask = State.get(BlockInMask);
4924 Value *ShuffledMask = State.Builder.CreateShuffleVector(
4927 "interleaved.mask");
4928 return MaskForGaps ? State.Builder.CreateBinOp(Instruction::And,
4929 ShuffledMask, MaskForGaps)
4933 const DataLayout &DL = Instr->getDataLayout();
4936 Value *MaskForGaps =
nullptr;
4940 assert(MaskForGaps &&
"Mask for Gaps is required but it is null");
4944 if (BlockInMask || MaskForGaps) {
4945 Value *GroupMask = CreateGroupMask(MaskForGaps);
4947 NewLoad = State.Builder.CreateMaskedLoad(VecTy, ResAddr,
4949 PoisonVec,
"wide.masked.vec");
4951 NewLoad = State.Builder.CreateAlignedLoad(VecTy, ResAddr,
4958 if (VecTy->isScalableTy()) {
4961 assert(InterleaveFactor <= 8 &&
4962 "Unsupported deinterleave factor for scalable vectors");
4963 NewLoad = State.Builder.CreateIntrinsicWithoutFolding(
4966 nullptr,
"strided.vec");
4969 auto CreateStridedVector = [&InterleaveFactor, &State,
4970 &NewLoad](
unsigned Index) ->
Value * {
4971 assert(Index < InterleaveFactor &&
"Illegal group index");
4972 if (State.VF.isScalable())
4973 return State.Builder.CreateExtractValue(NewLoad, Index);
4979 return State.Builder.CreateShuffleVector(NewLoad, StrideMask,
4983 for (
unsigned I = 0, J = 0;
I < InterleaveFactor; ++
I) {
4990 Value *StridedVec = CreateStridedVector(
I);
4993 if (Member->getType() != ScalarTy) {
5000 StridedVec = State.Builder.CreateVectorReverse(StridedVec,
"reverse");
5002 State.set(VPDefs[J], StridedVec);
5012 Value *MaskForGaps =
5015 "Mismatch between NeedsMaskForGaps and MaskForGaps");
5019 unsigned StoredIdx = 0;
5020 for (
unsigned i = 0; i < InterleaveFactor; i++) {
5022 "Fail to get a member from an interleaved store group");
5032 Value *StoredVec = State.get(StoredValues[StoredIdx]);
5036 StoredVec = State.Builder.CreateVectorReverse(StoredVec,
"reverse");
5040 if (StoredVec->
getType() != SubVT)
5049 if (BlockInMask || MaskForGaps) {
5050 Value *GroupMask = CreateGroupMask(MaskForGaps);
5051 NewStoreInstr = State.Builder.CreateMaskedStore(
5052 IVec, ResAddr, Group->
getAlign(), GroupMask);
5055 State.Builder.CreateAlignedStore(IVec, ResAddr, Group->
getAlign());
5062#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5066 O << Indent <<
"INTERLEAVE-GROUP with factor " << IG->getFactor() <<
", ";
5075 for (
unsigned i = 0; i < IG->getFactor(); ++i) {
5076 if (!IG->getMember(i))
5079 O <<
"\n" << Indent <<
" store ";
5081 O <<
" to index " << i;
5083 O <<
"\n" << Indent <<
" ";
5085 O <<
" = load from index " << i;
5093 assert(State.VF.isScalable() &&
5094 "Only support scalable VF for EVL tail-folding.");
5096 "Masking gaps for scalable vectors is not yet supported.");
5102 unsigned InterleaveFactor = Group->
getFactor();
5103 assert(InterleaveFactor <= 8 &&
5104 "Unsupported deinterleave/interleave factor for scalable vectors");
5111 Value *InterleaveEVL = State.Builder.CreateMul(
5112 EVL, ConstantInt::get(EVL->
getType(), InterleaveFactor),
"interleave.evl",
5116 Value *GroupMask =
nullptr;
5122 State.Builder.CreateVectorSplat(WideVF, State.Builder.getTrue());
5127 CallInst *NewLoad = State.Builder.CreateIntrinsicWithoutFolding(
5128 VecTy, Intrinsic::vp_load, {ResAddr, GroupMask, InterleaveEVL},
nullptr,
5139 NewLoad = State.Builder.CreateIntrinsicWithoutFolding(
5142 nullptr,
"strided.vec");
5144 const DataLayout &DL = Instr->getDataLayout();
5145 for (
unsigned I = 0, J = 0;
I < InterleaveFactor; ++
I) {
5151 Value *StridedVec = State.Builder.CreateExtractValue(NewLoad,
I);
5153 if (Member->getType() != ScalarTy) {
5171 const DataLayout &DL = Instr->getDataLayout();
5172 for (
unsigned I = 0, StoredIdx = 0;
I < InterleaveFactor;
I++) {
5180 Value *StoredVec = State.get(StoredValues[StoredIdx]);
5182 if (StoredVec->
getType() != SubVT)
5191 CallInst *NewStore = State.Builder.CreateIntrinsicWithoutFolding(
5193 {IVec, ResAddr, GroupMask, InterleaveEVL});
5203#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5207 O << Indent <<
"INTERLEAVE-GROUP with factor " << IG->getFactor() <<
", ";
5217 for (
unsigned i = 0; i < IG->getFactor(); ++i) {
5218 if (!IG->getMember(i))
5221 O <<
"\n" << Indent <<
" vp.store ";
5223 O <<
" to index " << i;
5225 O <<
"\n" << Indent <<
" ";
5227 O <<
" = vp.load from index " << i;
5238 unsigned InsertPosIdx = 0;
5239 for (
unsigned Idx = 0; IG->getFactor(); ++Idx)
5240 if (
auto *Member = IG->getMember(Idx)) {
5241 if (Member == InsertPos)
5253 unsigned InterleaveFactor = IG->getFactor();
5258 for (
unsigned IF = 0; IF < InterleaveFactor; IF++)
5259 if (IG->getMember(IF))
5264 InsertPos->
getOpcode(), WideVecTy, IG->getFactor(), Indices,
5265 IG->getAlign(), AS, Ctx.CostKind,
getMask(), NeedsMaskForGaps);
5267 if (!IG->isReverse())
5270 return Cost + IG->getNumMembers() *
5272 VectorTy, VectorTy, Ctx.CostKind, {},
5282 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
5290#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5294 "unexpected number of operands");
5295 O << Indent <<
"EMIT ";
5297 O <<
" = WIDEN-POINTER-INDUCTION ";
5313 O << Indent <<
"EMIT ";
5315 O <<
" = EXPAND SCEV " << *Expr;
5321#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5324 O << Indent <<
"EMIT ";
5326 O <<
" = WIDEN-CANONICAL-INDUCTION";
5333 auto &Builder = State.Builder;
5337 Type *VecTy = State.VF.isScalar()
5338 ? VectorInit->getType()
5342 State.CFG.VPBB2IRBB.at(
getParent()->getCFGPredecessor(0));
5343 if (State.VF.isVector()) {
5345 auto *One = ConstantInt::get(IdxTy, 1);
5348 auto *RuntimeVF =
getRuntimeVF(Builder, IdxTy, State.VF);
5349 auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
5350 VectorInit = Builder.CreateInsertElement(
5356 Phi->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
5357 Phi->addIncoming(VectorInit, VectorPH);
5358 State.set(
this, Phi);
5365 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
5370#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5373 O << Indent <<
"FIRST-ORDER-RECURRENCE-PHI ";
5390 State.CFG.VPBB2IRBB.at(
getParent()->getCFGPredecessor(0));
5391 bool ScalarPHI = State.VF.isScalar() ||
isInLoop();
5392 Value *StartV = State.get(StartVPV, ScalarPHI);
5396 assert(State.CurrentParentLoop->getHeader() == HeaderBB &&
5397 "recipe must be in the vector loop header");
5402 Phi->addIncoming(StartV, VectorPH);
5405#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5408 O << Indent <<
"WIDEN-REDUCTION-PHI ";
5424 "Blend should propagate first lane use");
5431 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
5434#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5437 O << Indent <<
"CONDITIONAL-INDUCTION-PHI ";
5451 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
5454#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5457 O << Indent <<
"WIDEN-PHI ";
5467 State.CFG.VPBB2IRBB.at(
getParent()->getCFGPredecessor(0));
5470 State.Builder.CreatePHI(StartMask->
getType(), 2,
"active.lane.mask");
5471 Phi->addIncoming(StartMask, VectorPH);
5472 State.set(
this, Phi);
5475#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5478 O << Indent <<
"ACTIVE-LANE-MASK-PHI ";
5486#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5489 O << Indent <<
"CURRENT-ITERATION-PHI ";
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
AMDGPU Lower Kernel Arguments
AMDGPU Register Bank Select
This file declares a class to represent arbitrary precision floating point values and provide a varie...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static const Function * getParent(const Value *V)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static void replaceAllUsesWith(Value *Old, Value *New, SmallPtrSet< BasicBlock *, 32 > &FreshBBs, bool IsHuge)
Replace all old uses with new ones, and push the updated BBs into FreshBBs.
Value * getPointer(Value *Ptr)
static constexpr Value * getValue(Ty &ValueOrUse)
static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
This file provides a LoopVectorizationPlanner class.
static const SCEV * getAddressAccessSCEV(Value *Ptr, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
Gets the address access SCEV for Ptr, if it should be used for cost modeling according to isAddressSC...
static const Function * getCalledFunction(const Value *V)
static bool isOrdered(const Instruction *I)
uint64_t IntrinsicInst * II
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue GlueChain, SDNodeFlags Flags)
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
This file contains the declarations of different VPlan-related auxiliary helpers.
static Value * interleaveVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vals, const Twine &Name)
Return a vector containing interleaved elements from multiple smaller input vectors.
static AttributeSet getCalleeFnAttrs(const VPInstruction &VPI)
Return the function attributes of the callee of VPI, which must be a call or intrinsic VPInstruction.
static const ConstantFP * getConstantFP(const VPValue *V)
Returns the ConstantFP V wraps, or nullptr if it does not wrap one.
static SmallVector< unsigned > getExtractValueIndices(ArrayRef< VPValue * > Ops)
static void executePhiRecipe(VPSingleDefRecipe *R, VPPhiAccessors &Phi, VPTransformState &State, bool IsScalar, const Twine &Name)
Shared execute logic for VPPhi and VPWidenPHIRecipe.
static Value * createBitOrPointerCast(IRBuilderBase &Builder, Value *V, VectorType *DstVTy, const DataLayout &DL)
static Instruction::BinaryOps getSubRecurOpcode(RecurKind Kind)
static Function * getCalledFunction(ArrayRef< VPValue * > Operands)
For call VPInstruction operands, return the called function.
static VPExecutionFrequency getExecutionFrequencyFromMD(const MDNode *Node)
Returns the execution frequency recorded in Node.
static void printRecurrenceKind(raw_ostream &OS, const RecurKind &Kind)
static unsigned getCalledFnOperandIndex(ArrayRef< VPValue * > Operands)
For call VPInstruction operands, return the operand index of the called function.
This file contains the declarations of the Vectorization Plan base classes:
static const fltSemantics & IEEEdouble()
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
bool contains(Attribute::AttrKind A) const
Return true if the builder has the specified attribute.
This class holds the attributes for a particular argument, parameter, function, or return value.
bool hasAttributes() const
Return true if attributes exists in this set.
bool hasAttribute(Attribute::AttrKind Kind) const
Return true if the attribute exists in this set.
LLVM_ABI std::string getAsString(bool InAttrGrp=false) const
static LLVM_ABI AttributeSet get(LLVMContext &C, const AttrBuilder &B)
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
static LLVM_ABI Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
LLVM Basic Block Representation.
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static LLVM_ABI bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)
Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_ULT
unsigned less than
static LLVM_ABI StringRef getPredicateName(Predicate P)
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
ConstantFP - Floating Point Values [float, double].
bool isNegZero() const
Return true if the value is negative zero.
bool isOne() const
Returns true if this value is exactly +1.0.
bool isZero() const
Return true if the value is positive or negative zero.
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
LLVM_ABI IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
static DebugLoc getUnknown()
constexpr bool isVector() const
One or more elements.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
constexpr bool isScalar() const
Exactly one element.
static bool isSupportedFloatingPointType(Type *Ty)
Returns true if Ty is a supported floating-point type for phi, select, or call FPMathOperators.
Convenience struct for specifying and reasoning about fast-math flags.
LLVM_ABI void print(raw_ostream &O) const
Print fast-math flags to O.
void setAllowContract(bool B=true)
bool noSignedZeros() const
void setAllowReciprocal(bool B=true)
bool allowReciprocal() const
void setNoSignedZeros(bool B=true)
bool allowReassoc() const
Flag queries.
void setNoNaNs(bool B=true)
void setAllowReassoc(bool B=true)
Flag setters.
void setApproxFunc(bool B=true)
void setNoInfs(bool B=true)
bool allowContract() const
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Class to represent function types.
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
bool willReturn() const
Determine if the function will return.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
AttributeList getAttributes() const
Return the attribute list for this Function.
bool doesNotThrow() const
Determine if the function cannot unwind.
Type * getReturnType() const
Returns the type of the ret val.
Represents flags for the getelementptr instruction/expression.
static GEPNoWrapFlags none()
Common base class shared among various IRBuilders.
const DataLayout & getDataLayout() const
Get data layout.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
LLVM_ABI Value * CreateVectorSpliceRight(Value *V1, Value *V2, Value *Offset, const Twine &Name="")
Create a vector.splice.right intrinsic call, or a shufflevector that produces the same result if the ...
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
CondBrInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
LLVM_ABI Value * CreateSelectFMF(Value *C, Value *True, Value *False, FMFSource FMFSource, const Twine &Name="", Instruction *MDFrom=nullptr)
LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
LLVM_ABI CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
LLVM_ABI Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateFreeze(Value *V, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Value * CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx, const Twine &Name="")
Create a call to the vector.extract intrinsic.
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
LLVM_ABI Value * CreateVectorReverse(Value *V, const Twine &Name="")
Return a vector value that contains the vector V reversed.
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Value * CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name="", Instruction *MDFrom=nullptr)
LLVM_ABI Value * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateCountTrailingZeroElems(Type *ResTy, Value *Mask, bool ZeroIsPoison=true, const Twine &Name="")
Create a call to llvm.experimental_cttz_elts.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
LLVM_ABI Value * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > OverloadTypes, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="", ArrayRef< OperandBundleDef > OpBundles={}, function_ref< void(CallInst *)> SetFn=[](CallInst *) {})
Variant to create a possibly constant-folded intrinsic.
LLVM_ABI CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getFalse()
Get the constant value for i1 false.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateLogicalOr(Value *Cond1, Value *Cond2, const Twine &Name="", Instruction *MDFrom=nullptr)
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)
LLVM_ABI Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
Value * CreateSExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a SExt or Trunc from the integer value V to DestTy.
Value * CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec, Value *Idx, const Twine &Name="")
Create a call to the vector.insert intrinsic.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
LLVM_ABI Value * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *Op, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
A struct for saving information about induction variables.
@ IK_FpInduction
Floating point induction variable.
@ IK_IntInduction
Integer induction variable. Step = C.
static InstructionCost getInvalid(CostType Val=0)
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
LLVM_ABI void dropPoisonGeneratingFlags()
Drops flags that may cause this instruction to evaluate to poison despite having non-poison inputs.
LLVM_ABI void dropPoisonGeneratingAttributes()
Drops attributes that may generate poison.
LLVM_ABI void dropUBImplyingAttrsAndUnknownMetadata(ArrayRef< unsigned > KnownIDs={})
This function drops non-debug unknown metadata (through dropUnknownNonDebugMetadata).
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
InstTy * getInsertPos() const
void addMetadata(InstTy *NewInst) const
Add metadata (e.g.
This is an important class for using LLVM in a threaded context.
Represents a single loop in the control flow graph.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Information for memory intrinsic cost model.
bool doesNotAccessMemory() const
Whether this function accesses no memory.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
static LLVM_ABI unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
unsigned getOpcode() const
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static LLVM_ABI bool isSubRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is for a sub operation.
static bool isFindIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
This class represents an analyzed expression in the program.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
This class represents the LLVM 'select' instruction.
This class provides computation of slot numbers for LLVM Assembly writing.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
bool isByteTy() const
True if this is an instance of ByteType.
bool isVectorTy() const
True if this is an instance of VectorType.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
bool isPointerTy() const
True if this is an instance of PointerType.
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isVoidTy() const
Return true if this is 'void'.
value_op_iterator value_op_end()
void setOperand(unsigned i, Value *Val)
Value * getOperand(unsigned i) const
value_op_iterator value_op_begin()
unsigned getNumOperands() const
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenMemoryRecipe.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account when normalized the first incoming value wi...
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isNormalized() const
A normalized blend is one that has an odd number of operands, whereby the first operand does not have...
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
const VPBlocksTy & getPredecessors() const
static bool isHeader(const VPBlockBase *VPB, const VPDominatorTree &VPDT)
Returns true if VPB is a loop header, based on regions or VPDT in their absence.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPBranchOnMaskRecipe.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
ArrayRef< VPRecipeValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
InductionDescriptor::InductionKind getInductionKind() const
VPValue * getIndex() const
VPValue * getStepValue() const
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPDerivedIVRecipe.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStartValue() const
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPExpandSCEVRecipe(const SCEV *Expr, bool ExpandInIR=false)
bool isVectorToScalar() const
Returns true if this VPExpressionRecipe produces a single scalar.
SmallVector< VPSingleDefRecipe * > decompose()
Return and insert the recipes of the expression back into the VPlan, directly before the current reci...
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
bool mayHaveSideEffects() const
Returns true if this expression contains recipes that may have side effects.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...
bool mayReadOrWriteMemory() const
Returns true if this expression contains recipes that may read from or write to memory.
VPExpressionRecipe(ExpressionTypes ExpressionType, ArrayRef< VPSingleDefRecipe * > ExpressionRecipes)
Construct a new VPExpressionRecipe by internalizing recipes in ExpressionRecipes.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a vectorized histogram operation.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPHistogramRecipe.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getMask() const
Return the mask operand if one was provided, or a null pointer if all lanes should be executed uncond...
void printFnAttrs(raw_ostream &O) const
Print function attributes carried with this object.
void printRetAttrs(raw_ostream &O) const
Print return-value attributes carried with this object.
MemoryEffects getMemoryEffects(AttributeSet CalleeFnAttrs) const
Returns the memory effects of a call with these attributes to a callee with function attributes Calle...
void intersectAttrs(const VPIRAttributes &Other, LLVMContext &Ctx)
Keep only the attributes valid for both this and Other.
void printParamAttrs(raw_ostream &O, unsigned ArgIdx) const
Print per-arg attributes for argument ArgIdx.
void applyAttrs(CallInst &WideCall) const
Apply the stored attributes to the widened call WideCall.
Class to record and manage LLVM IR flags.
ReductionFlagsTy ReductionFlags
LLVM_ABI_FOR_TEST bool flagsValidForOpcode(unsigned Opcode) const
Returns true if the set flags are valid for Opcode.
void printFlags(raw_ostream &O) const
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
static LLVM_ABI_FOR_TEST VPIRFlags getDefaultFlags(unsigned Opcode, Type *ResultTy=nullptr)
Returns default flags for Opcode and scalar ResultTy for opcodes that support it, asserts otherwise.
bool isReductionOrdered() const
CmpInst::Predicate getPredicate() const
LLVM_ABI_FOR_TEST FastMathFlags getFastMathFlagsOrNone() const
void intersectFlags(const VPIRFlags &Other)
Only keep flags also present in Other.
GEPNoWrapFlags getGEPNoWrapFlags() const
bool hasPredicate() const
Returns true if the recipe has a comparison predicate.
LLVM_ABI_FOR_TEST bool hasRequiredFlagsForOpcode(unsigned Opcode, Type *ResultTy) const
Returns true if Opcode with scalar result type ResultTy has its required flags set.
DisjointFlagsTy DisjointFlags
NonNegFlagsTy NonNegFlags
bool isReductionInLoop() const
void applyFlags(Instruction &I) const
Apply the IR flags to I.
RecurKind getRecurKind() const
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
LLVM_ABI_FOR_TEST InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPIRInstruction.
VPIRInstruction(Instruction &I)
VPIRInstruction::create() should be used to create VPIRInstructions, as subclasses may need to be cre...
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This is a concrete Recipe that models a single VPlan-level instruction.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPInstruction.
bool doesGeneratePerAllLanes() const
Returns true if this recipe produces scalar values for all VF lanes.
@ ExtractLastActive
Extracts the last active lane from a set of vectors.
@ Intrinsic
Calls a scalar intrinsic. The intrinsic ID is the last operand.
@ ExtractLane
Extracts a single lane (first operand) from a set of vector operands.
@ ExitingIVValue
Compute the exiting value of a wide induction after vectorization, that is the value of the last lane...
@ WideIVStep
Scale the first operand (vector step) by the second operand (scalar-step).
@ ExtractPenultimateElement
@ ResumeForEpilogue
Explicit user for values in the main VPlan, used by the epilogue vector loop.
@ Unpack
Extracts all lanes from its (non-scalable) vector operand.
@ FirstOrderRecurrenceSplice
@ ReductionStartVector
Start vector for reductions with 3 operands: the original start value, the identity value for the red...
@ BuildVector
Creates a fixed-width vector containing all operands.
@ CanonicalIVIncrementForPart
@ ComputeReductionResult
Reduce the operands to the final reduction result using the operation specified via the operation's V...
bool opcodeMayReadOrWriteFromMemory() const
Returns true if the underlying opcode may read from or write to memory.
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
StringRef getName() const
Returns the symbolic name assigned to the VPInstruction.
unsigned getOpcode() const
bool usesFirstPartOnly() const override
Returns true if the recipe only uses the first part of all operands.
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
void addOperand(VPValue *Op)
Add Op as operand of this VPInstruction.
VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands, const VPIRFlags &Flags={}, const VPIRMetadata &MD={}, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="", Type *ResultTy=nullptr, const VPIRAttributes &Attrs={})
bool isVectorToScalar() const
Returns true if this VPInstruction produces a scalar value from a vector, e.g.
VPValue * getMask() const
Returns the mask for the VPInstruction.
bool isSingleScalar() const
Returns true if the recipe produces a single scalar value.
unsigned getNumOperandsForOpcode() const
Return the number of operands determined by the opcode of the VPInstruction, excluding mask.
bool isMasked() const
Returns true if the VPInstruction has a mask operand.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
bool needsMaskForGaps() const
Return true if the access needs a mask because of the gaps.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this recipe.
Instruction * getInsertPos() const
const InterleaveGroup< Instruction > * getInterleaveGroup() const
VPValue * getMask() const
Return the mask used by this recipe.
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
VPValue * getAddr() const
Return the address accessed by this recipe.
VPValue * getEVL() const
The VPValue of the explicit vector length.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
unsigned getNumStoreOperands() const override
Returns the number of stored operands of this interleave group.
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
unsigned getNumStoreOperands() const override
Returns the number of stored operands of this interleave group.
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
static VPLane getLastLaneForVF(const ElementCount &VF)
static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset)
static VPLane getFirstLane()
Helper type to provide functions to access incoming values and blocks for phi-like recipes.
virtual const VPRecipeBase * getAsRecipe() const =0
Return a VPRecipeBase* to the current object.
LLVM_ABI_FOR_TEST VPValue * getIncomingValueForBlock(const VPBasicBlock *VPBB) const
Returns the incoming value for VPBB. VPBB must be an incoming block.
void removeIncomingValueFor(VPBlockBase *IncomingBlock) const
Removes the incoming value for IncomingBlock, which must be a predecessor.
detail::zippy< llvm::detail::zip_first, VPUser::const_operand_range, const_incoming_blocks_range > incoming_values_and_blocks() const
Returns an iterator range over pairs of incoming values and corresponding incoming blocks.
VPValue * getIncomingValue(unsigned Idx) const
Returns the incoming VPValue with index Idx.
void printPhiOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the recipe.
void setIncomingValueForBlock(const VPBasicBlock *VPBB, VPValue *V) const
Sets the incoming value for VPBB to V.
void execute(VPTransformState &State) override
Generates phi nodes for live-outs (from a replicate region) as needed to retain SSA form.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
void moveBefore(RecipeListTy::iterator I)
Unlink this recipe and insert it before I.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
virtual void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Each concrete VPRecipe prints itself, without printing common information, like debug info or metadat...
VPRegionBlock * getRegion()
RecipeListTy::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void dump() const
Dump the recipe to stderr (for debugging).
bool isPhi() const
Returns true for PHI-like recipes.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
VPRecipeTy getVPRecipeID() const
virtual InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const
Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
bool isSafeToSpeculativelyExecute() const
Return true if we can safely execute this recipe unconditionally even if it is masked originally.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
VPRecipeBase(VPRecipeTy SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this recipe, taking into account if the cost computation should be skipped and the...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const
Print the recipe, delegating to printRecipe().
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Type * getScalarType() const
Returns the scalar type of this VPRecipeValue.
void execute(VPTransformState &State) override
Generate the reduction in the loop.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getEVL() const
The VPValue of the explicit vector length.
unsigned getVFScaleFactor() const
Get the factor that the VF of this recipe's output should be scaled by, or 1 if it isn't scaled.
bool isInLoop() const
Returns true if the phi is part of an in-loop reduction.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool isConditional() const
Return true if the in-loop reduction is conditional.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of VPReductionRecipe.
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
VPValue * getCondOp() const
The VPValue of the condition for the block.
RecurKind getRecurrenceKind() const
Return the recurrence kind for the in-loop reduction.
bool isPartialReduction() const
Returns true if the reduction outputs a vector with a scaled down VF.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
bool isInLoop() const
Returns true if the reduction is in-loop.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the reduction in the loop.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
const VPBranchOnMaskRecipe * getEntryBranchOnMask() const
Return the VPBranchOnMaskRecipe from the entry block of this replicating region.
void execute(VPTransformState &State) override
Generate replicas of the desired Ingredient.
bool isSingleScalar() const
Returns true if the recipe produces a single scalar value.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPReplicateRecipe.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
static Type * computeScalarType(const Instruction *I, ArrayRef< VPValue * > Operands)
Compute the scalar result type for a VPReplicateRecipe wrapping I with Operands (excluding any predic...
static InstructionCost computeCallCost(Function *CalledFn, Type *ResultTy, ArrayRef< const VPValue * > ArgOps, bool IsSingleScalar, ElementCount VF, VPCostContext &Ctx)
Return the cost of scalarizing a call to CalledFn with argument operands ArgOps for a given VF.
unsigned getOpcode() const
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPScalarIVStepsRecipe.
bool doesGeneratePerAllLanes() const
Returns true if this recipe produces scalar values for all VF lanes.
VPValue * getStepValue() const
VPValue * getStartIndex() const
Return the StartIndex, or null if known to be zero, valid only after unrolling.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
VPSingleDefRecipe is a base class for recipes that model a sequence of one or more output IR that def...
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
VPSingleDefRecipe(VPRecipeTy SC, ArrayRef< VPValue * > Operands, Type *ResultTy, Value *UV=nullptr, DebugLoc DL=DebugLoc::getUnknown())
LLVM_DUMP_METHOD void dump() const
Print this VPSingleDefRecipe to dbgs() (for debugging).
This class can be used to assign names to VPValues.
A symbolic live-in VPValue, used for values like vector trip count, VF, and VFxUF.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
unsigned getNumOperands() const
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
bool operands_empty() const
VPValue * getLastOperand() const
Returns the last operand.
void addOperand(VPValue *Operand)
This is the base class of the VPlan Def/Use graph, used for modeling the data flow into,...
Type * getScalarType() const
Returns the scalar type of this VPValue, dispatching based on the concrete subclass.
Value * getLiveInIRValue() const
Return the underlying IR value for a VPIRValue.
bool isDefinedOutsideLoopRegions() const
Returns true if the VPValue is defined outside any loop.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
void setUnderlyingValue(Value *Val)
VPUser * getSingleUser()
Return the single user of this value, or nullptr if there is not exactly one user.
VPValue * getVFValue() const
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getSourceElementType() const
int64_t getStride() const
void materializeOffset(unsigned Part=0)
Adds the offset operand to the recipe.
VPValue * getStride() const
Type * getSourceElementType() const
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
VPValue * getVFxPart() const
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Function * getCalledScalarFunction() const
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenCallRecipe.
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
static InstructionCost computeCallCost(Function *Variant, VPCostContext &Ctx)
Return the cost of widening a call using the vector function Variant.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Instruction::CastOps getOpcode() const
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce widened copies of the cast.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenCastRecipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
Type * getSourceElementType() const
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
VPValue * getStepValue()
Returns the step value of the induction.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenIntOrFpInductionRecipe.
bool isTruncated() const
Returns true if this induction is evaluated in a narrower type than the start value.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
CallInst * createVectorCall(VPTransformState &State)
Helper function to produce the widened intrinsic call.
Intrinsic::ID getVectorIntrinsicID() const
Return the ID of the intrinsic.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
StringRef getIntrinsicName() const
Return to name of the intrinsic as string.
static InstructionCost computeCallCost(Intrinsic::ID ID, ArrayRef< const VPValue * > Operands, const VPRecipeWithIRFlags &R, ElementCount VF, VPCostContext &Ctx)
Compute the cost of a vector intrinsic with ID and Operands.
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
void execute(VPTransformState &State) override
Produce a widened version of the vector intrinsic.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this vector intrinsic.
static InstructionCost computeMemIntrinsicCost(Intrinsic::ID IID, Type *Ty, bool IsMasked, Align Alignment, VPCostContext &Ctx)
Helper function for computing the cost of vector memory intrinsic.
Type * getAccessType(ElementCount VF) const
Returns the data type of the memory access at VF.
void execute(VPTransformState &State) override
Produce a widened version of the vector memory intrinsic.
unsigned getAddrOpIdx() const
Returns the address operand index of the VPWidenMemIntrinsicRecipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this vector memory intrinsic.
VPValue * getMask() const
Returns the mask of a predicated VPWidenMemIntrinsicRecipe.
bool IsMasked
Whether the memory access is masked.
bool isConsecutive() const
Return whether the loaded-from / stored-to addresses are consecutive.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const
Return the cost of this VPWidenMemoryRecipe.
bool Consecutive
Whether the accessed addresses are consecutive.
VPValue * getMask() const
Return the mask used by this recipe.
Align Alignment
Alignment information for this memory access.
virtual VPRecipeBase * getAsRecipe()=0
Return a VPRecipeBase* to the current object.
Align getAlign() const
Returns the alignment of the memory access.
VPValue * getAddr() const
Return the address accessed by this recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenPHIRecipe.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool onlyScalarsGenerated(bool IsScalable)
Returns true if only scalar values will be generated.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenPointerInductionRecipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenRecipe.
void execute(VPTransformState &State) override
Produce a widened instruction using the opcode and operands of the recipe, processing State....
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
unsigned getOpcode() const
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
const DataLayout & getDataLayout() const
VPIRValue * getConstantInt(Type *Ty, uint64_t Val, bool IsSigned=false)
Return a VPIRValue wrapping a ConstantInt with the given type and value.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
LLVMContext & getContext() const
All values hold a context through their type.
LLVM_ABI void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
void mutateType(Type *Ty)
Mutate the type of this Value to be of the specified type.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
static LLVM_ABI bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
Type * getElementType() const
static BlockMass getFull()
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
const ilist_detail::compute_node_options< VPRecipeBase, Options... >::type::parent_ty * getParent() const
void setParent(ilist_detail::compute_node_options< VPRecipeBase, Options... >::type::parent_ty *Parent)
self_iterator getIterator()
typename base_list_type::iterator iterator
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ BasicBlock
Various leaf nodes.
LLVM_ABI Intrinsic::ID getDeinterleaveIntrinsicID(unsigned Factor)
Returns the corresponding llvm.vector.deinterleaveN intrinsic for factor N.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
LLVM_ABI AttributeSet getFnAttributes(LLVMContext &C, ID id)
Return the function attributes for an intrinsic.
LLVM_ABI StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
match_combine_or< Ty... > m_CombineOr(const Ty &...Ps)
Combine pattern matchers matching any of Ps patterns.
auto m_Cmp()
Matches any compare instruction and ignore it.
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
LogicalOp_match< LHS, RHS, Instruction::And, true > m_c_LogicalAnd(const LHS &L, const RHS &R)
Matches L && R with LHS and RHS in either order.
LogicalOp_match< LHS, RHS, Instruction::Or, true > m_c_LogicalOr(const LHS &L, const RHS &R)
Matches L || R with LHS and RHS in either order.
auto m_ZExtOrTrunc(const Op0_t &Op0)
int_pred_ty< is_zero_int, 1 > m_False()
auto m_VPValue()
Match an arbitrary VPValue and ignore it.
VPInstruction_match< VPInstruction::ExplicitVectorLength, Op0_t > m_EVL(const Op0_t &Op0)
int_pred_ty< is_one, 1 > m_True()
VPInstruction_match< VPInstruction::BranchOnCond > m_BranchOnCond()
VPInstruction_match< VPInstruction::Reverse, Op0_t > m_Reverse(const Op0_t &Op0)
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
NodeAddr< DefNode * > Def
friend class Instruction
Iterator for Instructions in a `BasicBlock.
BranchProbability getExecutionProbability(BlockFrequency Freq)
Returns Freq as a BranchProbability, relative to the full mass.
bool isSingleScalar(const VPValue *VPV)
Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...
bool isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE, const Loop *L)
Returns true if Addr is an address SCEV that can be passed to TTI::getAddressComputationCost,...
Intrinsic::ID getIntrinsicID(const Ty *R)
Return the intrinsic ID underlying a call.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used by all users.
bool propagatesFirstLaneUse(const VPUser *U, const VPValue *Op)
Returns true if checking whether U uses the first lane of Op requires recursively checking Op's users...
bool onlyScalarValuesUsed(const VPValue *Def)
Returns true if only scalar values of Def are used by all users.
bool isUsedByLoadStoreAddress(const VPValue *V)
Returns true if V is used as part of the address of another load or store.
LLVM_ABI_FOR_TEST const SCEV * getSCEVExprForVPValue(const VPValue *V, PredicatedScalarEvolution &PSE, const Loop *L=nullptr)
Return the SCEV expression for V.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
LLVM_ABI Value * createSimpleReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind)
Create a reduction of the given vector.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
@ Undef
Value of the register doesn't matter.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
LLVM_ABI std::optional< unsigned > getVectorStoreIntrinsicDataArgIdx(Intrinsic::ID ID)
Returns the argument index of the data value of the vector store intrinsic ID, or std::nullopt if the...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
VectorInstrContext
Represents a hint about the context in which a vector instruction or intrinsic is used.
@ None
The instruction is not folded.
@ BinaryOp
One of the operands is a binary op.
VPBuilderBase<> VPBuilder
auto map_to_vector(ContainerTy &&C, FuncTy &&F)
Map a range to a SmallVector with element types deduced from the mapping.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
auto cast_or_null(const Y &Val)
LLVM_ABI Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
bool isa_and_nonnull(const Y &Val)
LLVM_ABI Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)
Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
auto dyn_cast_or_null(const Y &Val)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI Constant * createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, const InterleaveGroup< Instruction > &Group)
Create a mask that filters the members of an interleave group where there are gaps.
LLVM_ABI llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
auto reverse(ContainerTy &&C)
ElementCount getVectorizedTypeVF(Type *Ty)
Returns the number of vector elements for a vectorized type.
LLVM_ABI llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)
Create a mask with replicated elements.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isPointerTy(const Type *T)
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
Type * toVectorizedTy(Type *Ty, ElementCount EC)
A helper for converting to vectorized types.
LLVM_ABI Type * computeScalarTypeForInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands)
Compute the scalar result type for an IR Opcode given Operands.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
LLVM_ABI bool isVectorIntrinsicWithStructReturnOverloadAtField(Intrinsic::ID ID, int RetIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic that returns a struct is overloaded at the struct elem...
static const MachineInstrBuilder & addOffset(const MachineInstrBuilder &MIB, int Offset)
LLVM_ABI llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
LLVM_ABI std::optional< unsigned > getVectorMemoryIntrinsicPointerArgIdx(Intrinsic::ID ID)
Returns the argument index of the pointer parameter for the vector memory intrinsic ID,...
RecurKind
These are the kinds of recurrences that we support.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ FMinimumNum
FP min with llvm.minimumnum semantics.
@ FindIV
FindIV reduction with select(icmp(),x,y) where one of (x,y) is a loop induction variable (increasing ...
@ Or
Bitwise or logical OR of integers.
@ FMinimum
FP min with llvm.minimum semantics.
@ FMaxNum
FP max with llvm.maxnum semantics including NaNs.
@ Mul
Product of integers.
@ FSub
Subtraction of floats.
@ FAddChainWithSubs
A chain of fadds and fsubs.
@ AnyOf
AnyOf reduction with select(cmp(),x,y) where one of (x,y) is loop invariant, and both x and y are int...
@ Xor
Bitwise or logical XOR of integers.
@ FindLast
FindLast reduction with select(cmp(),x,y) where x and y.
@ FMax
FP max implemented in terms of select(cmp()).
@ FMaximum
FP max with llvm.maximum semantics.
@ FMulAdd
Sum of float products with llvm.fmuladd(a * b + sum).
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
@ FMin
FP min implemented in terms of select(cmp()).
@ FMinNum
FP min with llvm.minnum semantics including NaNs.
@ Sub
Subtraction of integers.
@ AddChainWithSubs
A chain of adds and subs.
@ FMaximumNum
FP max with llvm.maximumnum semantics.
@ UMax
Unsigned integer max implemented in terms of select(cmp()).
LLVM_ABI bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic has a scalar operand.
LLVM_ABI Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)
Given information about an recurrence kind, return the identity for the @llvm.vector....
DWARFExpression::Operation Op
LLVM_ABI std::optional< unsigned > getVectorIntrinsicMaskArgIdx(Intrinsic::ID ID)
Returns the argument index of the mask for the vector intrinsic ID, or std::nullopt if the intrinsic ...
LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Extract branch weights from MD_prof metadata.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
LLVM_ABI Value * createOrderedReduction(IRBuilderBase &B, RecurKind RdxKind, Value *Src, Value *Start)
Create an ordered reduction intrinsic using the given recurrence kind RdxKind.
ArrayRef< Type * > getContainedTypes(Type *const &Ty)
Returns the types contained in Ty.
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
LLVM_ABI bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Struct to hold various analysis needed for cost computations.
static bool isFreeScalarIntrinsic(Intrinsic::ID ID)
Returns true if ID is a pseudo intrinsic that is dropped via scalarization rather than widened.
static bool executesAtMostOnce(const VPlan &Plan, ElementCount VF)
Returns true if the vector loop body of Plan is known to execute at most once at VF,...
TargetTransformInfo::TargetCostKind CostKind
The frequency with which a recipe executes, relative to the entry of the loop region.
void execute(VPTransformState &State) override
Generate the phi nodes.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this first-order recurrence phi recipe.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use cast/dyn_cast/isa and exec...
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
PHINode & getIRPhi() const
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost getCostForRecipeWithOpcode(unsigned Opcode, ElementCount VF, VPCostContext &Ctx) const
Compute the cost for this recipe for VF, using Opcode and Ctx.
VPRecipeWithIRFlags(VPRecipeTy SC, ArrayRef< VPValue * > Operands, Type *ResultTy, const VPIRFlags &Flags, DebugLoc DL=DebugLoc::getUnknown())
void execute(VPTransformState &State) override
Generate the wide load or gather.
VPRecipeBase * getAsRecipe() override
Return a VPRecipeBase* to the current object.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenLoadEVLRecipe.
VPValue * getEVL() const
Return the EVL operand.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate a wide load or gather.
VPRecipeBase * getAsRecipe() override
Return a VPRecipeBase* to the current object.
VPValue * getStoredValue() const
Return the address accessed by this recipe.
void execute(VPTransformState &State) override
Generate the wide store or scatter.
VPRecipeBase * getAsRecipe() override
Return a VPRecipeBase* to the current object.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenStoreEVLRecipe.
VPValue * getEVL() const
Return the EVL operand.
void execute(VPTransformState &State) override
Generate a wide store or scatter.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase * getAsRecipe() override
Return a VPRecipeBase* to the current object.
VPValue * getStoredValue() const
Return the value stored by this recipe.