36#define DEBUG_TYPE "vector-combine"
42STATISTIC(NumVecLoad,
"Number of vector loads formed");
43STATISTIC(NumVecCmp,
"Number of vector compares formed");
44STATISTIC(NumVecBO,
"Number of vector binops formed");
45STATISTIC(NumVecCmpBO,
"Number of vector compare + binop formed");
46STATISTIC(NumShufOfBitcast,
"Number of shuffles moved after bitcast");
47STATISTIC(NumScalarBO,
"Number of scalar binops formed");
48STATISTIC(NumScalarCmp,
"Number of scalar compares formed");
52 cl::desc(
"Disable all vector combine transforms"));
56 cl::desc(
"Disable binop extract to shuffle transforms"));
60 cl::desc(
"Max number of instructions to scan for vector combining."));
62static const unsigned InvalidIndex = std::numeric_limits<unsigned>::max();
70 :
F(
F), Builder(
F.getContext()),
TTI(
TTI), DT(DT), AA(AA), AC(AC),
DL(
DL),
71 TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
86 bool TryEarlyFoldsOnly;
97 unsigned PreferredExtractIndex)
const;
101 unsigned PreferredExtractIndex);
118 bool foldSelectShuffle(
Instruction &
I,
bool FromReduction =
false);
122 if (
auto *NewI = dyn_cast<Instruction>(&New)) {
142 while (
auto *BitCast = dyn_cast<BitCastInst>(V))
143 V = BitCast->getOperand(0);
151 if (!Load || !Load->isSimple() || !Load->hasOneUse() ||
152 Load->getFunction()->hasFnAttribute(Attribute::SanitizeMemTag) ||
158 Type *ScalarTy = Load->getType()->getScalarType();
161 if (!ScalarSize || !MinVectorSize || MinVectorSize % ScalarSize != 0 ||
168bool VectorCombine::vectorizeLoadInsert(
Instruction &
I) {
182 auto *
Load = dyn_cast<LoadInst>(
X);
194 Value *SrcPtr =
Load->getPointerOperand()->stripPointerCasts();
195 assert(isa<PointerType>(SrcPtr->
getType()) &&
"Expected a pointer type");
197 unsigned MinVecNumElts = MinVectorSize / ScalarSize;
198 auto *MinVecTy = VectorType::get(ScalarTy, MinVecNumElts,
false);
199 unsigned OffsetEltIndex = 0;
207 unsigned OffsetBitWidth =
DL->getIndexTypeSizeInBits(SrcPtr->
getType());
218 uint64_t ScalarSizeInBytes = ScalarSize / 8;
219 if (
Offset.urem(ScalarSizeInBytes) != 0)
223 OffsetEltIndex =
Offset.udiv(ScalarSizeInBytes).getZExtValue();
224 if (OffsetEltIndex >= MinVecNumElts)
241 unsigned AS =
Load->getPointerAddressSpace();
260 auto *Ty = cast<FixedVectorType>(
I.getType());
261 unsigned OutputNumElts = Ty->getNumElements();
263 assert(OffsetEltIndex < MinVecNumElts &&
"Address offset too big");
264 Mask[0] = OffsetEltIndex;
270 if (OldCost < NewCost || !NewCost.
isValid())
281 replaceValue(
I, *VecLd);
291 auto *Shuf = cast<ShuffleVectorInst>(&
I);
292 if (!Shuf->isIdentityWithPadding())
297 cast<FixedVectorType>(Shuf->getOperand(0)->getType())->getNumElements();
298 unsigned OpIndex =
any_of(Shuf->getShuffleMask(), [&NumOpElts](
int M) {
299 return M >= (int)(NumOpElts);
302 auto *
Load = dyn_cast<LoadInst>(Shuf->getOperand(
OpIndex));
309 auto *Ty = cast<FixedVectorType>(
I.getType());
310 Value *SrcPtr =
Load->getPointerOperand()->stripPointerCasts();
311 assert(isa<PointerType>(SrcPtr->
getType()) &&
"Expected a pointer type");
318 unsigned AS =
Load->getPointerAddressSpace();
333 if (OldCost < NewCost || !NewCost.
isValid())
340 replaceValue(
I, *VecLd);
352 assert(Index0C && Index1C &&
"Expected constant extract indexes");
354 unsigned Index0 = Index0C->getZExtValue();
355 unsigned Index1 = Index1C->getZExtValue();
358 if (Index0 == Index1)
383 if (PreferredExtractIndex == Index0)
385 if (PreferredExtractIndex == Index1)
389 return Index0 > Index1 ? Ext0 : Ext1;
401 unsigned PreferredExtractIndex) {
402 auto *Ext0IndexC = dyn_cast<ConstantInt>(Ext0->
getOperand(1));
403 auto *Ext1IndexC = dyn_cast<ConstantInt>(Ext1->
getOperand(1));
404 assert(Ext0IndexC && Ext1IndexC &&
"Expected constant extract indexes");
406 unsigned Opcode =
I.getOpcode();
417 assert((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
418 "Expected a compare");
428 unsigned Ext0Index = Ext0IndexC->getZExtValue();
429 unsigned Ext1Index = Ext1IndexC->getZExtValue();
444 InstructionCost CheapExtractCost = std::min(Extract0Cost, Extract1Cost);
454 bool HasUseTax = Ext0 == Ext1 ? !Ext0->
hasNUses(2)
456 OldCost = CheapExtractCost + ScalarOpCost;
457 NewCost = VectorOpCost + CheapExtractCost + HasUseTax * CheapExtractCost;
461 OldCost = Extract0Cost + Extract1Cost + ScalarOpCost;
462 NewCost = VectorOpCost + CheapExtractCost +
467 ConvertToShuffle = getShuffleExtract(Ext0, Ext1, PreferredExtractIndex);
468 if (ConvertToShuffle) {
486 return OldCost < NewCost;
496 auto *VecTy = cast<FixedVectorType>(Vec->
getType());
498 ShufMask[NewIndex] = OldIndex;
517 assert(isa<ConstantInt>(
C) &&
"Expected a constant index operand");
518 if (isa<Constant>(
X))
531 assert(isa<CmpInst>(&
I) &&
"Expected a compare");
534 "Expected matching constant extract indexes");
542 replaceValue(
I, *NewExt);
550 assert(isa<BinaryOperator>(&
I) &&
"Expected a binary operator");
553 "Expected matching constant extract indexes");
563 if (
auto *VecBOInst = dyn_cast<Instruction>(VecBO))
564 VecBOInst->copyIRFlags(&
I);
567 replaceValue(
I, *NewExt);
595 auto *Ext0 = cast<ExtractElementInst>(I0);
596 auto *Ext1 = cast<ExtractElementInst>(I1);
603 if (isExtractExtractCheap(Ext0, Ext1,
I, ExtractToChange, InsertIndex))
606 if (ExtractToChange) {
607 unsigned CheapExtractIdx = ExtractToChange == Ext0 ? C1 : C0;
612 if (ExtractToChange == Ext0)
619 foldExtExtCmp(Ext0, Ext1,
I);
621 foldExtExtBinop(Ext0, Ext1,
I);
648 auto *VecTy = cast<FixedVectorType>(
I.getType());
649 if (SrcVec->
getType() != VecTy)
653 unsigned NumElts = VecTy->getNumElements();
654 if (
Index >= NumElts)
661 std::iota(
Mask.begin(),
Mask.end(), 0);
680 if (NewCost > OldCost)
687 replaceValue(
I, *Shuf);
706 auto *DestTy = dyn_cast<FixedVectorType>(
I.getType());
707 auto *SrcTy = dyn_cast<FixedVectorType>(V0->
getType());
708 if (!DestTy || !SrcTy)
711 unsigned DestEltSize = DestTy->getScalarSizeInBits();
712 unsigned SrcEltSize = SrcTy->getScalarSizeInBits();
713 if (SrcTy->getPrimitiveSizeInBits() % DestEltSize != 0)
716 bool IsUnary = isa<UndefValue>(V1);
723 if (!(BCTy0 && BCTy0->getElementType() == DestTy->getElementType()) &&
724 !(BCTy1 && BCTy1->getElementType() == DestTy->getElementType()))
729 if (DestEltSize <= SrcEltSize) {
732 assert(SrcEltSize % DestEltSize == 0 &&
"Unexpected shuffle mask");
733 unsigned ScaleFactor = SrcEltSize / DestEltSize;
738 assert(DestEltSize % SrcEltSize == 0 &&
"Unexpected shuffle mask");
739 unsigned ScaleFactor = DestEltSize / SrcEltSize;
746 unsigned NumSrcElts = SrcTy->getPrimitiveSizeInBits() / DestEltSize;
751 unsigned NumOps = IsUnary ? 1 : 2;
763 TargetTransformInfo::CastContextHint::None,
768 TargetTransformInfo::CastContextHint::None, CK);
769 if (DestCost > SrcCost || !DestCost.
isValid())
777 replaceValue(
I, *Shuf);
784bool VectorCombine::scalarizeVPIntrinsic(
Instruction &
I) {
785 if (!isa<VPIntrinsic>(
I))
798 if (!ScalarOp0 || !ScalarOp1)
806 auto IsAllTrueMask = [](
Value *MaskVal) {
808 if (
auto *ConstValue = dyn_cast<Constant>(SplattedVal))
809 return ConstValue->isAllOnesValue();
825 if (
auto *FVTy = dyn_cast<FixedVectorType>(VecTy))
826 Mask.resize(FVTy->getNumElements(), 0);
834 Args.push_back(
V->getType());
840 std::optional<unsigned> FunctionalOpcode =
842 std::optional<Intrinsic::ID> ScalarIntrID = std::nullopt;
843 if (!FunctionalOpcode) {
867 <<
", Cost of scalarizing:" << NewCost <<
"\n");
870 if (OldCost < NewCost || !NewCost.
isValid())
881 bool SafeToSpeculate;
884 .
hasFnAttr(Attribute::AttrKind::Speculatable);
887 *FunctionalOpcode, &VPI,
nullptr, &AC, &DT);
888 if (!SafeToSpeculate &&
895 {ScalarOp0, ScalarOp1})
897 ScalarOp0, ScalarOp1);
905bool VectorCombine::scalarizeBinopOrCmp(
Instruction &
I) {
916 bool IsCmp = Pred != CmpInst::Predicate::BAD_ICMP_PREDICATE;
918 for (
User *U :
I.users())
928 Constant *VecC0 =
nullptr, *VecC1 =
nullptr;
929 Value *V0 =
nullptr, *V1 =
nullptr;
942 if (IsConst0 && IsConst1)
944 if (!IsConst0 && !IsConst1 && Index0 != Index1)
949 auto *I0 = dyn_cast_or_null<Instruction>(V0);
950 auto *
I1 = dyn_cast_or_null<Instruction>(V1);
951 if ((IsConst0 && I1 &&
I1->mayReadFromMemory()) ||
957 Type *VecTy =
I.getType();
962 "Unexpected types for insert element into binop or cmp");
964 unsigned Opcode =
I.getOpcode();
983 (IsConst0 ? 0 : InsertCost) + (IsConst1 ? 0 : InsertCost) + VectorOpCost;
985 (IsConst0 ? 0 : !Ins0->
hasOneUse() * InsertCost) +
986 (IsConst1 ? 0 : !Ins1->
hasOneUse() * InsertCost);
989 if (OldCost < NewCost || !NewCost.
isValid())
1009 Scalar->setName(
I.getName() +
".scalar");
1013 if (
auto *ScalarInst = dyn_cast<Instruction>(Scalar))
1014 ScalarInst->copyIRFlags(&
I);
1018 IsCmp ? Builder.
CreateCmp(Pred, VecC0, VecC1)
1021 replaceValue(
I, *Insert);
1031 if (!
I.isBinaryOp() || !
I.getType()->isIntegerTy(1))
1037 Value *B0 =
I.getOperand(0), *B1 =
I.getOperand(1);
1055 auto *Ext0 = cast<ExtractElementInst>(I0);
1056 auto *Ext1 = cast<ExtractElementInst>(I1);
1065 : Instruction::ICmp;
1066 auto *VecTy = dyn_cast<FixedVectorType>(
X->getType());
1083 int CheapIndex = ConvertToShuf == Ext0 ? Index1 : Index0;
1084 int ExpensiveIndex = ConvertToShuf == Ext0 ? Index0 : Index1;
1089 ShufMask[CheapIndex] = ExpensiveIndex;
1098 if (OldCost < NewCost || !NewCost.
isValid())
1112 replaceValue(
I, *NewExt);
1121 unsigned NumScanned = 0;
1131class ScalarizationResult {
1132 enum class StatusTy { Unsafe, Safe, SafeWithFreeze };
1137 ScalarizationResult(StatusTy
Status,
Value *ToFreeze =
nullptr)
1141 ScalarizationResult(
const ScalarizationResult &
Other) =
default;
1142 ~ScalarizationResult() {
1143 assert(!ToFreeze &&
"freeze() not called with ToFreeze being set");
1146 static ScalarizationResult unsafe() {
return {StatusTy::Unsafe}; }
1147 static ScalarizationResult safe() {
return {StatusTy::Safe}; }
1148 static ScalarizationResult safeWithFreeze(
Value *ToFreeze) {
1149 return {StatusTy::SafeWithFreeze, ToFreeze};
1153 bool isSafe()
const {
return Status == StatusTy::Safe; }
1155 bool isUnsafe()
const {
return Status == StatusTy::Unsafe; }
1158 bool isSafeWithFreeze()
const {
return Status == StatusTy::SafeWithFreeze; }
1163 Status = StatusTy::Unsafe;
1168 assert(isSafeWithFreeze() &&
1169 "should only be used when freezing is required");
1171 "UserI must be a user of ToFreeze");
1177 if (
U.get() == ToFreeze)
1194 uint64_t NumElements = VecTy->getElementCount().getKnownMinValue();
1196 if (
auto *
C = dyn_cast<ConstantInt>(
Idx)) {
1197 if (
C->getValue().ult(NumElements))
1198 return ScalarizationResult::safe();
1199 return ScalarizationResult::unsafe();
1202 unsigned IntWidth =
Idx->getType()->getScalarSizeInBits();
1203 APInt Zero(IntWidth, 0);
1204 APInt MaxElts(IntWidth, NumElements);
1210 true, &AC, CtxI, &DT)))
1211 return ScalarizationResult::safe();
1212 return ScalarizationResult::unsafe();
1225 if (ValidIndices.
contains(IdxRange))
1226 return ScalarizationResult::safeWithFreeze(IdxBase);
1227 return ScalarizationResult::unsafe();
1237 if (
auto *
C = dyn_cast<ConstantInt>(
Idx))
1239 C->getZExtValue() *
DL.getTypeStoreSize(ScalarType));
1251bool VectorCombine::foldSingleElementStore(
Instruction &
I) {
1252 auto *
SI = cast<StoreInst>(&
I);
1253 if (!
SI->isSimple() || !isa<VectorType>(
SI->getValueOperand()->getType()))
1261 if (!
match(
SI->getValueOperand(),
1266 if (
auto *Load = dyn_cast<LoadInst>(Source)) {
1267 auto VecTy = cast<VectorType>(
SI->getValueOperand()->getType());
1268 Value *SrcAddr =
Load->getPointerOperand()->stripPointerCasts();
1271 if (!
Load->isSimple() ||
Load->getParent() !=
SI->getParent() ||
1272 !
DL->typeSizeEqualsStoreSize(
Load->getType()->getScalarType()) ||
1273 SrcAddr !=
SI->getPointerOperand()->stripPointerCasts())
1277 if (ScalarizableIdx.isUnsafe() ||
1282 if (ScalarizableIdx.isSafeWithFreeze())
1283 ScalarizableIdx.freeze(Builder, *cast<Instruction>(
Idx));
1285 SI->getValueOperand()->getType(),
SI->getPointerOperand(),
1286 {ConstantInt::get(Idx->getType(), 0), Idx});
1293 replaceValue(
I, *NSI);
1302bool VectorCombine::scalarizeLoadExtract(
Instruction &
I) {
1307 auto *VecTy = cast<VectorType>(
I.getType());
1308 auto *LI = cast<LoadInst>(&
I);
1309 if (LI->isVolatile() || !
DL->typeSizeEqualsStoreSize(VecTy->
getScalarType()))
1314 LI->getPointerAddressSpace());
1318 unsigned NumInstChecked = 0;
1322 for (
auto &Pair : NeedFreeze)
1323 Pair.second.discard();
1330 auto *UI = dyn_cast<ExtractElementInst>(U);
1331 if (!UI || UI->getParent() != LI->getParent())
1338 make_range(std::next(LI->getIterator()), UI->getIterator())) {
1345 LastCheckedInst = UI;
1349 if (ScalarIdx.isUnsafe())
1351 if (ScalarIdx.isSafeWithFreeze()) {
1353 ScalarIdx.discard();
1356 auto *
Index = dyn_cast<ConstantInt>(UI->getOperand(1));
1363 Align(1), LI->getPointerAddressSpace());
1367 if (ScalarizedCost >= OriginalCost)
1372 auto *EI = cast<ExtractElementInst>(U);
1376 auto It = NeedFreeze.
find(EI);
1377 if (It != NeedFreeze.
end())
1378 It->second.freeze(Builder, *cast<Instruction>(
Idx));
1383 auto *NewLoad = cast<LoadInst>(Builder.
CreateLoad(
1384 VecTy->getElementType(),
GEP, EI->getName() +
".scalar"));
1387 LI->getAlign(), VecTy->getElementType(),
Idx, *
DL);
1388 NewLoad->setAlignment(ScalarOpAlignment);
1390 replaceValue(*EI, *NewLoad);
1393 FailureGuard.release();
1399bool VectorCombine::foldShuffleOfBinops(
Instruction &
I) {
1400 auto *VecTy = cast<FixedVectorType>(
I.getType());
1416 if (ShufCost > BinopCost)
1422 if (BinaryOperator::isCommutative(Opcode) &&
X != Z &&
Y != W)
1425 Value *Shuf0, *Shuf1;
1430 }
else if (
Y == W) {
1440 if (
auto *NewInst = dyn_cast<Instruction>(NewBO)) {
1441 NewInst->copyIRFlags(B0);
1442 NewInst->andIRFlags(B1);
1446 replaceValue(
I, *NewBO);
1452bool VectorCombine::foldShuffleOfCastops(
Instruction &
I) {
1459 auto *C0 = dyn_cast<CastInst>(V0);
1460 auto *C1 = dyn_cast<CastInst>(V1);
1465 if (C0->getSrcTy() != C1->getSrcTy())
1469 if (Opcode != C1->getOpcode()) {
1471 Opcode = Instruction::SExt;
1476 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(
I.getType());
1477 auto *CastDstTy = dyn_cast<FixedVectorType>(C0->getDestTy());
1478 auto *CastSrcTy = dyn_cast<FixedVectorType>(C0->getSrcTy());
1479 if (!ShuffleDstTy || !CastDstTy || !CastSrcTy)
1482 unsigned NumSrcElts = CastSrcTy->getNumElements();
1483 unsigned NumDstElts = CastDstTy->getNumElements();
1484 assert((NumDstElts == NumSrcElts || Opcode == Instruction::BitCast) &&
1485 "Only bitcasts expected to alter src/dst element counts");
1489 if (NumDstElts != NumSrcElts && (NumSrcElts % NumDstElts) != 0 &&
1490 (NumDstElts % NumSrcElts) != 0)
1494 if (NumSrcElts >= NumDstElts) {
1497 assert(NumSrcElts % NumDstElts == 0 &&
"Unexpected shuffle mask");
1498 unsigned ScaleFactor = NumSrcElts / NumDstElts;
1503 assert(NumDstElts % NumSrcElts == 0 &&
"Unexpected shuffle mask");
1504 unsigned ScaleFactor = NumDstElts / NumSrcElts;
1509 auto *NewShuffleDstTy =
1522 OldMask,
CostKind, 0,
nullptr, std::nullopt, &
I);
1530 <<
"\n OldCost: " << OldCost <<
" vs NewCost: " << NewCost
1532 if (NewCost > OldCost)
1540 if (
auto *NewInst = dyn_cast<Instruction>(Cast)) {
1541 NewInst->copyIRFlags(C0);
1542 NewInst->andIRFlags(C1);
1546 replaceValue(
I, *Cast);
1553bool VectorCombine::foldShuffleFromReductions(
Instruction &
I) {
1554 auto *II = dyn_cast<IntrinsicInst>(&
I);
1557 switch (II->getIntrinsicID()) {
1558 case Intrinsic::vector_reduce_add:
1559 case Intrinsic::vector_reduce_mul:
1560 case Intrinsic::vector_reduce_and:
1561 case Intrinsic::vector_reduce_or:
1562 case Intrinsic::vector_reduce_xor:
1563 case Intrinsic::vector_reduce_smin:
1564 case Intrinsic::vector_reduce_smax:
1565 case Intrinsic::vector_reduce_umin:
1566 case Intrinsic::vector_reduce_umax:
1575 std::queue<Value *> Worklist;
1578 if (
auto *
Op = dyn_cast<Instruction>(
I.getOperand(0)))
1581 while (!Worklist.empty()) {
1582 Value *CV = Worklist.front();
1593 if (
auto *CI = dyn_cast<Instruction>(CV)) {
1594 if (CI->isBinaryOp()) {
1595 for (
auto *
Op : CI->operand_values())
1598 }
else if (
auto *SV = dyn_cast<ShuffleVectorInst>(CI)) {
1599 if (Shuffle && Shuffle != SV)
1616 for (
auto *V : Visited)
1617 for (
auto *U :
V->users())
1618 if (!Visited.contains(U) && U != &
I)
1622 dyn_cast<FixedVectorType>(II->getOperand(0)->getType());
1627 if (!ShuffleInputType)
1635 sort(ConcatMask, [](
int X,
int Y) {
return (
unsigned)
X < (
unsigned)
Y; });
1639 bool IsTruncatingShuffle =
VecType->getNumElements() < NumInputElts;
1640 bool UsesSecondVec =
1641 any_of(ConcatMask, [&](
int M) {
return M >= (int)NumInputElts; });
1644 (UsesSecondVec && !IsTruncatingShuffle) ? VecType : ShuffleInputType;
1650 VecTyForCost, ConcatMask);
1652 LLVM_DEBUG(
dbgs() <<
"Found a reduction feeding from a shuffle: " << *Shuffle
1654 LLVM_DEBUG(
dbgs() <<
" OldCost: " << OldCost <<
" vs NewCost: " << NewCost
1656 if (NewCost < OldCost) {
1660 LLVM_DEBUG(
dbgs() <<
"Created new shuffle: " << *NewShuffle <<
"\n");
1661 replaceValue(*Shuffle, *NewShuffle);
1666 return foldSelectShuffle(*Shuffle,
true);
1671bool VectorCombine::foldTruncFromReductions(
Instruction &
I) {
1672 auto *II = dyn_cast<IntrinsicInst>(&
I);
1678 case Intrinsic::vector_reduce_add:
1679 case Intrinsic::vector_reduce_mul:
1680 case Intrinsic::vector_reduce_and:
1681 case Intrinsic::vector_reduce_or:
1682 case Intrinsic::vector_reduce_xor:
1689 Value *ReductionSrc =
I.getOperand(0);
1695 auto *Trunc = cast<CastInst>(ReductionSrc);
1696 auto *TruncSrcTy = cast<VectorType>(TruncSrc->
getType());
1697 auto *ReductionSrcTy = cast<VectorType>(ReductionSrc->
getType());
1698 Type *ResultTy =
I.getType();
1710 ReductionSrcTy->getScalarType(),
1713 if (OldCost <= NewCost || !NewCost.
isValid())
1717 TruncSrcTy->getScalarType(), II->getIntrinsicID(), {TruncSrc});
1719 replaceValue(
I, *NewTruncation);
1733bool VectorCombine::foldSelectShuffle(
Instruction &
I,
bool FromReduction) {
1734 auto *SVI = cast<ShuffleVectorInst>(&
I);
1735 auto *VT = cast<FixedVectorType>(
I.getType());
1736 auto *Op0 = dyn_cast<Instruction>(SVI->getOperand(0));
1737 auto *Op1 = dyn_cast<Instruction>(SVI->getOperand(1));
1738 if (!Op0 || !Op1 || Op0 == Op1 || !Op0->isBinaryOp() || !Op1->isBinaryOp() ||
1742 auto *SVI0A = dyn_cast<Instruction>(Op0->getOperand(0));
1743 auto *SVI0B = dyn_cast<Instruction>(Op0->getOperand(1));
1744 auto *SVI1A = dyn_cast<Instruction>(Op1->getOperand(0));
1745 auto *SVI1B = dyn_cast<Instruction>(Op1->getOperand(1));
1748 if (!
I ||
I->getOperand(0)->getType() != VT)
1751 return U != Op0 && U != Op1 &&
1752 !(isa<ShuffleVectorInst>(U) &&
1753 (InputShuffles.contains(cast<Instruction>(U)) ||
1754 isInstructionTriviallyDead(cast<Instruction>(U))));
1757 if (checkSVNonOpUses(SVI0A) || checkSVNonOpUses(SVI0B) ||
1758 checkSVNonOpUses(SVI1A) || checkSVNonOpUses(SVI1B))
1766 for (
auto *U :
I->users()) {
1767 auto *SV = dyn_cast<ShuffleVectorInst>(U);
1768 if (!SV || SV->getType() != VT)
1770 if ((SV->getOperand(0) != Op0 && SV->getOperand(0) != Op1) ||
1771 (SV->getOperand(1) != Op0 && SV->getOperand(1) != Op1))
1778 if (!collectShuffles(Op0) || !collectShuffles(Op1))
1782 if (FromReduction && Shuffles.
size() > 1)
1787 if (!FromReduction) {
1789 for (
auto *U : SV->users()) {
1792 Shuffles.push_back(SSV);
1804 int MaxV1Elt = 0, MaxV2Elt = 0;
1805 unsigned NumElts = VT->getNumElements();
1808 SVN->getShuffleMask(Mask);
1812 Value *SVOp0 = SVN->getOperand(0);
1813 Value *SVOp1 = SVN->getOperand(1);
1814 if (isa<UndefValue>(SVOp1)) {
1815 auto *SSV = cast<ShuffleVectorInst>(SVOp0);
1818 for (
unsigned I = 0, E =
Mask.size();
I != E;
I++) {
1824 if (SVOp0 == Op1 && SVOp1 == Op0) {
1828 if (SVOp0 != Op0 || SVOp1 != Op1)
1835 for (
unsigned I = 0;
I <
Mask.size();
I++) {
1838 }
else if (Mask[
I] <
static_cast<int>(NumElts)) {
1839 MaxV1Elt = std::max(MaxV1Elt, Mask[
I]);
1840 auto It =
find_if(V1, [&](
const std::pair<int, int> &
A) {
1841 return Mask[
I] ==
A.first;
1850 MaxV2Elt = std::max<int>(MaxV2Elt, Mask[
I] - NumElts);
1851 auto It =
find_if(V2, [&](
const std::pair<int, int> &
A) {
1852 return Mask[
I] -
static_cast<int>(NumElts) ==
A.first;
1855 ReconstructMask.
push_back(NumElts + It -
V2.begin());
1858 V2.emplace_back(Mask[
I] - NumElts, NumElts +
V2.size());
1866 sort(ReconstructMask);
1867 OrigReconstructMasks.
push_back(std::move(ReconstructMask));
1874 if (V1.
empty() ||
V2.empty() ||
1875 (MaxV1Elt ==
static_cast<int>(V1.
size()) - 1 &&
1876 MaxV2Elt ==
static_cast<int>(
V2.size()) - 1))
1883 auto *SV = dyn_cast<ShuffleVectorInst>(
I);
1886 if (isa<UndefValue>(SV->getOperand(1)))
1887 if (
auto *SSV = dyn_cast<ShuffleVectorInst>(SV->getOperand(0)))
1888 if (InputShuffles.contains(SSV))
1890 return SV->getMaskValue(M);
1898 std::pair<int, int>
Y) {
1899 int MXA = GetBaseMaskValue(
A,
X.first);
1900 int MYA = GetBaseMaskValue(
A,
Y.first);
1903 stable_sort(V1, [&](std::pair<int, int>
A, std::pair<int, int>
B) {
1904 return SortBase(SVI0A,
A,
B);
1906 stable_sort(V2, [&](std::pair<int, int>
A, std::pair<int, int>
B) {
1907 return SortBase(SVI1A,
A,
B);
1912 for (
const auto &Mask : OrigReconstructMasks) {
1914 for (
int M : Mask) {
1916 auto It =
find_if(V, [M](
auto A) {
return A.second ==
M; });
1917 assert(It !=
V.end() &&
"Expected all entries in Mask");
1918 return std::distance(
V.begin(), It);
1922 else if (M <
static_cast<int>(NumElts)) {
1923 ReconstructMask.
push_back(FindIndex(V1, M));
1925 ReconstructMask.
push_back(NumElts + FindIndex(V2, M));
1928 ReconstructMasks.push_back(std::move(ReconstructMask));
1934 for (
unsigned I = 0;
I < V1.
size();
I++) {
1935 V1A.
push_back(GetBaseMaskValue(SVI0A, V1[
I].first));
1936 V1B.
push_back(GetBaseMaskValue(SVI0B, V1[
I].first));
1938 for (
unsigned I = 0;
I <
V2.size();
I++) {
1939 V2A.
push_back(GetBaseMaskValue(SVI1A, V2[
I].first));
1940 V2B.
push_back(GetBaseMaskValue(SVI1B, V2[
I].first));
1942 while (V1A.
size() < NumElts) {
1946 while (V2A.
size() < NumElts) {
1952 auto *SV = dyn_cast<ShuffleVectorInst>(
I);
1958 VT, SV->getShuffleMask());
1969 CostBefore += std::accumulate(Shuffles.begin(), Shuffles.end(),
1971 CostBefore += std::accumulate(InputShuffles.begin(), InputShuffles.end(),
1983 CostAfter += std::accumulate(ReconstructMasks.begin(), ReconstructMasks.end(),
1985 std::set<SmallVector<int>> OutputShuffleMasks({V1A, V1B, V2A, V2B});
1987 std::accumulate(OutputShuffleMasks.begin(), OutputShuffleMasks.end(),
1990 LLVM_DEBUG(
dbgs() <<
"Found a binop select shuffle pattern: " <<
I <<
"\n");
1992 <<
" vs CostAfter: " << CostAfter <<
"\n");
1993 if (CostBefore <= CostAfter)
1998 auto *SV = dyn_cast<ShuffleVectorInst>(
I);
2001 if (isa<UndefValue>(SV->getOperand(1)))
2002 if (
auto *SSV = dyn_cast<ShuffleVectorInst>(SV->getOperand(0)))
2003 if (InputShuffles.contains(SSV))
2005 return SV->getOperand(
Op);
2009 GetShuffleOperand(SVI0A, 1), V1A);
2012 GetShuffleOperand(SVI0B, 1), V1B);
2015 GetShuffleOperand(SVI1A, 1), V2A);
2018 GetShuffleOperand(SVI1B, 1), V2B);
2022 if (
auto *
I = dyn_cast<Instruction>(NOp0))
2023 I->copyIRFlags(Op0,
true);
2027 if (
auto *
I = dyn_cast<Instruction>(NOp1))
2028 I->copyIRFlags(Op1,
true);
2030 for (
int S = 0, E = ReconstructMasks.size(); S != E; S++) {
2033 replaceValue(*Shuffles[S], *NSV);
2036 Worklist.pushValue(NSV0A);
2037 Worklist.pushValue(NSV0B);
2038 Worklist.pushValue(NSV1A);
2039 Worklist.pushValue(NSV1B);
2040 for (
auto *S : Shuffles)
2047bool VectorCombine::run() {
2055 bool MadeChange =
false;
2058 bool IsFixedVectorType = isa<FixedVectorType>(
I.getType());
2059 auto Opcode =
I.getOpcode();
2065 if (IsFixedVectorType) {
2067 case Instruction::InsertElement:
2068 MadeChange |= vectorizeLoadInsert(
I);
2070 case Instruction::ShuffleVector:
2071 MadeChange |= widenSubvectorLoad(
I);
2080 if (isa<VectorType>(
I.getType())) {
2081 MadeChange |= scalarizeBinopOrCmp(
I);
2082 MadeChange |= scalarizeLoadExtract(
I);
2083 MadeChange |= scalarizeVPIntrinsic(
I);
2086 if (Opcode == Instruction::Store)
2087 MadeChange |= foldSingleElementStore(
I);
2090 if (TryEarlyFoldsOnly)
2097 if (IsFixedVectorType) {
2099 case Instruction::InsertElement:
2100 MadeChange |= foldInsExtFNeg(
I);
2102 case Instruction::ShuffleVector:
2103 MadeChange |= foldShuffleOfBinops(
I);
2104 MadeChange |= foldShuffleOfCastops(
I);
2105 MadeChange |= foldSelectShuffle(
I);
2107 case Instruction::BitCast:
2108 MadeChange |= foldBitcastShuffle(
I);
2113 case Instruction::Call:
2114 MadeChange |= foldShuffleFromReductions(
I);
2115 MadeChange |= foldTruncFromReductions(
I);
2117 case Instruction::ICmp:
2118 case Instruction::FCmp:
2119 MadeChange |= foldExtractExtract(
I);
2123 MadeChange |= foldExtractExtract(
I);
2124 MadeChange |= foldExtractedCmps(
I);
2137 if (
I.isDebugOrPseudoInst())
2143 while (!Worklist.isEmpty()) {
2166 VectorCombine
Combiner(
F,
TTI, DT, AA, AC,
DL, TryEarlyFoldsOnly);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This is the interface for LLVM's primary stateless and local alias analysis.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static cl::opt< TargetTransformInfo::TargetCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(TargetTransformInfo::TCK_RecipThroughput), cl::values(clEnumValN(TargetTransformInfo::TCK_RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(TargetTransformInfo::TCK_Latency, "latency", "Instruction latency"), clEnumValN(TargetTransformInfo::TCK_CodeSize, "code-size", "Code size"), clEnumValN(TargetTransformInfo::TCK_SizeAndLatency, "size-latency", "Code size and latency")))
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
std::optional< std::vector< StOtherPiece > > Other
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This is the interface for a simple mod/ref and alias analysis over globals.
static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo, MemorySSAUpdater &MSSAU)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
FunctionAnalysisManager FAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static Value * createShiftShuffle(Value *Vec, unsigned OldIndex, unsigned NewIndex, IRBuilder<> &Builder)
Create a shuffle that translates (shifts) 1 element from the input vector to a new element location.
static Value * peekThroughBitcasts(Value *V)
Return the source operand of a potentially bitcasted value.
static Align computeAlignmentAfterScalarization(Align VectorAlignment, Type *ScalarType, Value *Idx, const DataLayout &DL)
The memory operation on a vector of ScalarType had alignment of VectorAlignment.
static ScalarizationResult canScalarizeAccess(VectorType *VecTy, Value *Idx, Instruction *CtxI, AssumptionCache &AC, const DominatorTree &DT)
Check if it is legal to scalarize a memory access to VecTy at index Idx.
static cl::opt< bool > DisableVectorCombine("disable-vector-combine", cl::init(false), cl::Hidden, cl::desc("Disable all vector combine transforms"))
static bool canWidenLoad(LoadInst *Load, const TargetTransformInfo &TTI)
static const unsigned InvalidIndex
static cl::opt< unsigned > MaxInstrsToScan("vector-combine-max-scan-instrs", cl::init(30), cl::Hidden, cl::desc("Max number of instructions to scan for vector combining."))
static cl::opt< bool > DisableBinopExtractShuffle("disable-binop-extract-shuffle", cl::init(false), cl::Hidden, cl::desc("Disable binop extract to shuffle transforms"))
static bool isMemModifiedBetween(BasicBlock::iterator Begin, BasicBlock::iterator End, const MemoryLocation &Loc, AAResults &AA)
static ExtractElementInst * translateExtract(ExtractElementInst *ExtElt, unsigned NewIndex, IRBuilder<> &Builder)
Given an extract element instruction with constant index operand, shuffle the source vector (shift th...
A manager for alias analyses.
ModRefInfo getModRefInfo(const Instruction *I, const std::optional< MemoryLocation > &OptLoc)
Check whether or not an instruction may read or write the optionally specified memory location.
Class for arbitrary precision integers.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
LLVM Basic Block Representation.
InstListType::iterator iterator
Instruction iterators...
BinaryOps getOpcode() const
Represents analyses that only rely on functions' control flow.
Value * getArgOperand(unsigned i) const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
bool isFPPredicate() const
static Constant * getExtractElement(Constant *Vec, Constant *Idx, Type *OnlyIfReducedTy=nullptr)
This is the shared class of boolean and integer constants.
const APInt & getValue() const
Return the constant as an APInt value reference.
This class represents a range of values.
ConstantRange urem(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an unsigned remainder operation of...
ConstantRange binaryAnd(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a binary-and of a value in this ra...
bool contains(const APInt &Val) const
Return true if the specified value is in the set.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
Analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateFNegFMF(Value *V, Instruction *FMFSource, const Twine &Name="")
Copy fast-math-flags from an instruction rather than using the builder's default FMF.
Value * CreateFreeze(Value *V, const Twine &Name="")
Value * CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
Value * CreatePointerBitCastOrAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
InstructionWorklist - This is the worklist management logic for InstCombine and other simplification ...
void pushUsersToWorkList(Instruction &I)
When an instruction is simplified, add all users of the instruction to the work lists because they mi...
void push(Instruction *I)
Push the instruction onto the worklist stack.
void remove(Instruction *I)
Remove I from the worklist if it exists.
bool comesBefore(const Instruction *Other) const
Given an instruction Other in the same basic block as this instruction, return true if this instructi...
bool mayReadFromMemory() const LLVM_READONLY
Return true if this instruction may read memory.
void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
An instruction for reading from memory.
Representation for a specific memory location.
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
This instruction constructs a fixed permutation of two input vectors.
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static void commuteShuffleMask(MutableArrayRef< int > Mask, unsigned InVecNumElts)
Change values in a shuffle permute mask assuming the two vector operands of length InVecNumElts have ...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
void setAlignment(Align Align)
Analysis pass providing the TargetTransformInfo.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
static bool isVPBinOp(Intrinsic::ID ID)
This is the common base class for vector predication intrinsics.
std::optional< unsigned > getFunctionalIntrinsicID() const
std::optional< unsigned > getFunctionalOpcode() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
const Value * stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, APInt &Offset) const
This is a wrapper around stripAndAccumulateConstantOffsets with the in-bounds requirement set to fals...
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
bool hasNUses(unsigned N) const
Return true if this Value has exactly N uses.
StringRef getName() const
Return a constant reference to the value's name.
PreservedAnalyses run(Function &F, FunctionAnalysisManager &)
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
AttributeList getAttributes(LLVMContext &C, ID id)
Return the attributes for an intrinsic.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
BinaryOp_match< LHS, RHS, Instruction::URem > m_URem(const LHS &L, const RHS &R)
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
match_combine_and< LTy, RTy > m_CombineAnd(const LTy &L, const RTy &R)
Combine two pattern matchers matching L && R.
CastOperator_match< OpTy, Instruction::Trunc > m_Trunc(const OpTy &Op)
Matches Trunc.
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
OneUse_match< T > m_OneUse(const T &SubPattern)
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
OneOps_match< OpTy, Instruction::Load > m_Load(const OpTy &Op)
Matches LoadInst.
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
match_combine_or< CastInst_match< OpTy, SExtInst >, NNegZExt_match< OpTy > > m_SExtLike(const OpTy &Op)
Match either "sext" or "zext nneg".
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
This is an optimization pass for GlobalISel generic memory operations.
void stable_sort(R &&Range)
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)
Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getArithmeticReductionInstruction(Intrinsic::ID RdxID)
Returns the arithmetic instruction opcode used when expanding a reduction.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool mustSuppressSpeculation(const LoadInst &LI)
Return true if speculation of the given load must be suppressed to avoid ordering or interfering with...
bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
ConstantRange computeConstantRange(const Value *V, bool ForSigned, bool UseInstrInfo=true, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Determine the possible constant range of an integer or vector of integer value.
bool isSafeToSpeculativelyExecuteWithOpcode(unsigned Opcode, const Instruction *Inst, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
This returns the same result as isSafeToSpeculativelyExecute if Opcode is the actual opcode of Inst.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
bool isModSet(const ModRefInfo MRI)
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
constexpr int PoisonMaskElem
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
DWARFExpression::Operation Op
bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isSafeToLoadUnconditionally(Value *V, Align Alignment, APInt &Size, const DataLayout &DL, Instruction *ScanFrom=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if we know that executing a load from this value cannot trap.
bool isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Returns true if V cannot be poison, but may be undef.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.