50#define DEBUG_TYPE "scalarizer"
54 cl::desc(
"Allow the scalarizer pass to scalarize "
55 "insertelement/extractelement with variable index"));
62 cl::desc(
"Allow the scalarizer pass to scalarize loads and store"));
71 cl::desc(
"Instruct the scalarizer pass to attempt to keep values of a "
72 "minimum number of bits"));
78 if (isa<PHINode>(Itr))
94using ScatterMap = std::map<std::pair<Value *, Type *>, ValueVector>;
105 unsigned NumPacked = 0;
109 unsigned NumFragments = 0;
112 Type *SplitTy =
nullptr;
116 Type *RemainderTy =
nullptr;
118 Type *getFragmentType(
unsigned I)
const {
119 return RemainderTy &&
I == NumFragments - 1 ? RemainderTy : SplitTy;
127 Scatterer() =
default;
133 const VectorSplit &VS, ValueVector *cachePtr =
nullptr);
136 Value *operator[](
unsigned I);
139 unsigned size()
const {
return VS.NumFragments; }
147 ValueVector *CachePtr;
154 FCmpSplitter(
FCmpInst &fci) : FCI(fci) {}
167 ICmpSplitter(
ICmpInst &ici) : ICI(ici) {}
179struct UnarySplitter {
191struct BinarySplitter {
207 Align getFragmentAlign(
unsigned Frag) {
225 unsigned NumElements =
VS.VecTy->getNumElements();
229 if (
VS.NumPacked > 1) {
232 ExtendMask.
resize(NumElements, -1);
233 for (
unsigned I = 0;
I <
VS.NumPacked; ++
I)
236 InsertMask.
resize(NumElements);
237 for (
unsigned I = 0;
I < NumElements; ++
I)
242 for (
unsigned I = 0;
I <
VS.NumFragments; ++
I) {
243 Value *Fragment = Fragments[
I];
245 unsigned NumPacked =
VS.NumPacked;
246 if (
I ==
VS.NumFragments - 1 &&
VS.RemainderTy) {
247 if (
auto *RemVecTy = dyn_cast<FixedVectorType>(
VS.RemainderTy))
248 NumPacked = RemVecTy->getNumElements();
253 if (NumPacked == 1) {
261 for (
unsigned J = 0; J < NumPacked; ++J)
262 InsertMask[
I *
VS.NumPacked + J] = NumElements + J;
265 for (
unsigned J = 0; J < NumPacked; ++J)
266 InsertMask[
I *
VS.NumPacked + J] =
I *
VS.NumPacked + J;
275T getWithDefaultOverride(
const cl::opt<T> &ClOption,
276 const std::optional<T> &DefaultOverride) {
278 : DefaultOverride.value_or(ClOption);
281class ScalarizerVisitor :
public InstVisitor<ScalarizerVisitor, bool> {
284 : DT(DT), ScalarizeVariableInsertExtract(getWithDefaultOverride(
286 Options.ScalarizeVariableInsertExtract)),
316 void gather(
Instruction *
Op,
const ValueVector &CV,
const VectorSplit &VS);
318 bool canTransferMetadata(
unsigned Kind);
319 void transferMetadataAndIRFlags(
Instruction *
Op,
const ValueVector &CV);
320 std::optional<VectorSplit> getVectorSplit(
Type *Ty);
321 std::optional<VectorLayout> getVectorLayout(
Type *Ty,
Align Alignment,
325 template<
typename T>
bool splitUnary(
Instruction &,
const T &);
326 template<
typename T>
bool splitBinary(
Instruction &,
const T &);
330 ScatterMap Scattered;
338 const bool ScalarizeVariableInsertExtract;
339 const bool ScalarizeLoadStore;
340 const unsigned ScalarizeMinBits;
346 const VectorSplit &VS, ValueVector *cachePtr)
347 : BB(bb), BBI(bbi),
V(
v),
VS(
VS), CachePtr(cachePtr) {
350 Tmp.resize(
VS.NumFragments,
nullptr);
352 assert((CachePtr->empty() ||
VS.NumFragments == CachePtr->size() ||
354 "Inconsistent vector sizes");
355 if (
VS.NumFragments > CachePtr->size())
356 CachePtr->resize(
VS.NumFragments,
nullptr);
361Value *Scatterer::operator[](
unsigned Frag) {
362 ValueVector &CV = CachePtr ? *CachePtr : Tmp;
372 V->getName() +
".i" +
Twine(Frag));
376 Type *FragmentTy =
VS.getFragmentType(Frag);
378 if (
auto *VecTy = dyn_cast<FixedVectorType>(FragmentTy)) {
380 for (
unsigned J = 0; J < VecTy->getNumElements(); ++J)
381 Mask.push_back(Frag *
VS.NumPacked + J);
384 V->getName() +
".i" +
Twine(Frag));
396 unsigned J =
Idx->getZExtValue();
398 if (Frag *
VS.NumPacked == J) {
399 CV[Frag] =
Insert->getOperand(1);
403 if (
VS.NumPacked == 1 && !CV[J]) {
407 CV[J] =
Insert->getOperand(1);
411 V->getName() +
".i" +
Twine(Frag));
417bool ScalarizerVisitor::visit(
Function &
F) {
418 assert(Gathered.empty() && Scattered.empty());
430 if (
Done &&
I->getType()->isVoidTy())
431 I->eraseFromParent();
440 const VectorSplit &VS) {
441 if (
Argument *VArg = dyn_cast<Argument>(V)) {
446 return Scatterer(BB, BB->
begin(), V, VS, &Scattered[{V, VS.SplitTy}]);
455 if (!DT->isReachableFromEntry(VOp->getParent()))
463 &Scattered[{
V,
VS.SplitTy}]);
474void ScalarizerVisitor::gather(
Instruction *
Op,
const ValueVector &CV,
475 const VectorSplit &VS) {
476 transferMetadataAndIRFlags(
Op, CV);
480 ValueVector &SV = Scattered[{
Op,
VS.SplitTy}];
482 for (
unsigned I = 0, E = SV.size();
I != E; ++
I) {
484 if (V ==
nullptr || SV[
I] == CV[
I])
488 if (isa<Instruction>(CV[
I]))
489 CV[
I]->takeName(Old);
491 PotentiallyDeadInstrs.emplace_back(Old);
495 Gathered.push_back(GatherList::value_type(
Op, &SV));
501 Op->replaceAllUsesWith(CV);
502 PotentiallyDeadInstrs.emplace_back(
Op);
509bool ScalarizerVisitor::canTransferMetadata(
unsigned Tag) {
510 return (
Tag == LLVMContext::MD_tbaa
511 ||
Tag == LLVMContext::MD_fpmath
512 ||
Tag == LLVMContext::MD_tbaa_struct
513 ||
Tag == LLVMContext::MD_invariant_load
514 ||
Tag == LLVMContext::MD_alias_scope
515 ||
Tag == LLVMContext::MD_noalias
516 ||
Tag == LLVMContext::MD_mem_parallel_loop_access
517 ||
Tag == LLVMContext::MD_access_group);
522void ScalarizerVisitor::transferMetadataAndIRFlags(
Instruction *
Op,
523 const ValueVector &CV) {
525 Op->getAllMetadataOtherThanDebugLoc(MDs);
526 for (
Value *V : CV) {
528 for (
const auto &MD : MDs)
529 if (canTransferMetadata(MD.first))
530 New->setMetadata(MD.first, MD.second);
531 New->copyIRFlags(
Op);
532 if (
Op->getDebugLoc() && !
New->getDebugLoc())
533 New->setDebugLoc(
Op->getDebugLoc());
539std::optional<VectorSplit> ScalarizerVisitor::getVectorSplit(
Type *Ty) {
541 Split.VecTy = dyn_cast<FixedVectorType>(Ty);
545 unsigned NumElems =
Split.VecTy->getNumElements();
546 Type *ElemTy =
Split.VecTy->getElementType();
551 Split.NumFragments = NumElems;
552 Split.SplitTy = ElemTy;
555 if (
Split.NumPacked >= NumElems)
561 unsigned RemainderElems = NumElems %
Split.NumPacked;
562 if (RemainderElems > 1)
564 else if (RemainderElems == 1)
565 Split.RemainderTy = ElemTy;
574std::optional<VectorLayout>
575ScalarizerVisitor::getVectorLayout(
Type *Ty,
Align Alignment,
577 std::optional<VectorSplit>
VS = getVectorSplit(Ty);
584 if (!
DL.typeSizeEqualsStoreSize(
VS->SplitTy) ||
585 (
VS->RemainderTy && !
DL.typeSizeEqualsStoreSize(
VS->RemainderTy)))
587 Layout.VecAlign = Alignment;
588 Layout.SplitSize =
DL.getTypeStoreSize(
VS->SplitTy);
594template<
typename Splitter>
595bool ScalarizerVisitor::splitUnary(
Instruction &
I,
const Splitter &Split) {
596 std::optional<VectorSplit>
VS = getVectorSplit(
I.getType());
600 std::optional<VectorSplit> OpVS;
601 if (
I.getOperand(0)->getType() ==
I.getType()) {
604 OpVS = getVectorSplit(
I.getOperand(0)->getType());
605 if (!OpVS ||
VS->NumPacked != OpVS->NumPacked)
610 Scatterer
Op = scatter(&
I,
I.getOperand(0), *OpVS);
611 assert(
Op.size() ==
VS->NumFragments &&
"Mismatched unary operation");
613 Res.resize(
VS->NumFragments);
614 for (
unsigned Frag = 0; Frag <
VS->NumFragments; ++Frag)
615 Res[Frag] =
Split(Builder,
Op[Frag],
I.getName() +
".i" +
Twine(Frag));
616 gather(&
I, Res, *VS);
622template<
typename Splitter>
623bool ScalarizerVisitor::splitBinary(
Instruction &
I,
const Splitter &Split) {
624 std::optional<VectorSplit>
VS = getVectorSplit(
I.getType());
628 std::optional<VectorSplit> OpVS;
629 if (
I.getOperand(0)->getType() ==
I.getType()) {
632 OpVS = getVectorSplit(
I.getOperand(0)->getType());
633 if (!OpVS ||
VS->NumPacked != OpVS->NumPacked)
638 Scatterer VOp0 = scatter(&
I,
I.getOperand(0), *OpVS);
639 Scatterer VOp1 = scatter(&
I,
I.getOperand(1), *OpVS);
640 assert(VOp0.size() ==
VS->NumFragments &&
"Mismatched binary operation");
641 assert(VOp1.size() ==
VS->NumFragments &&
"Mismatched binary operation");
643 Res.resize(
VS->NumFragments);
644 for (
unsigned Frag = 0; Frag <
VS->NumFragments; ++Frag) {
645 Value *Op0 = VOp0[Frag];
646 Value *Op1 = VOp1[Frag];
647 Res[Frag] =
Split(Builder, Op0, Op1,
I.getName() +
".i" +
Twine(Frag));
649 gather(&
I, Res, *VS);
659bool ScalarizerVisitor::splitCall(
CallInst &CI) {
660 std::optional<VectorSplit>
VS = getVectorSplit(CI.
getType());
675 ValueVector ScalarOperands(NumArgs);
686 for (
unsigned I = 0;
I != NumArgs; ++
I) {
688 if ([[maybe_unused]]
auto *OpVecTy =
689 dyn_cast<FixedVectorType>(OpI->
getType())) {
690 assert(OpVecTy->getNumElements() ==
VS->VecTy->getNumElements());
691 std::optional<VectorSplit> OpVS = getVectorSplit(OpI->
getType());
692 if (!OpVS || OpVS->NumPacked !=
VS->NumPacked) {
703 Scattered[
I] = scatter(&CI, OpI, *OpVS);
705 OverloadIdx[
I] = Tys.
size();
709 ScalarOperands[
I] = OpI;
715 ValueVector Res(
VS->NumFragments);
716 ValueVector ScalarCallOps(NumArgs);
722 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
723 bool IsRemainder =
I ==
VS->NumFragments - 1 &&
VS->RemainderTy;
724 ScalarCallOps.clear();
727 Tys[0] =
VS->RemainderTy;
729 for (
unsigned J = 0; J != NumArgs; ++J) {
731 ScalarCallOps.push_back(ScalarOperands[J]);
733 ScalarCallOps.push_back(Scattered[J][
I]);
734 if (IsRemainder && OverloadIdx[J] >= 0)
735 Tys[OverloadIdx[J]] = Scattered[J][
I]->getType();
742 Res[
I] = Builder.
CreateCall(NewIntrin, ScalarCallOps,
746 gather(&CI, Res, *VS);
750bool ScalarizerVisitor::visitSelectInst(
SelectInst &SI) {
751 std::optional<VectorSplit>
VS = getVectorSplit(
SI.getType());
755 std::optional<VectorSplit> CondVS;
756 if (isa<FixedVectorType>(
SI.getCondition()->getType())) {
757 CondVS = getVectorSplit(
SI.getCondition()->getType());
758 if (!CondVS || CondVS->NumPacked !=
VS->NumPacked) {
765 Scatterer VOp1 = scatter(&SI,
SI.getOperand(1), *VS);
766 Scatterer VOp2 = scatter(&SI,
SI.getOperand(2), *VS);
767 assert(VOp1.size() ==
VS->NumFragments &&
"Mismatched select");
768 assert(VOp2.size() ==
VS->NumFragments &&
"Mismatched select");
770 Res.resize(
VS->NumFragments);
773 Scatterer VOp0 = scatter(&SI,
SI.getOperand(0), *CondVS);
774 assert(VOp0.size() == CondVS->NumFragments &&
"Mismatched select");
775 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
784 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
791 gather(&SI, Res, *VS);
795bool ScalarizerVisitor::visitICmpInst(
ICmpInst &ICI) {
796 return splitBinary(ICI, ICmpSplitter(ICI));
799bool ScalarizerVisitor::visitFCmpInst(
FCmpInst &FCI) {
800 return splitBinary(FCI, FCmpSplitter(FCI));
803bool ScalarizerVisitor::visitUnaryOperator(
UnaryOperator &UO) {
804 return splitUnary(UO, UnarySplitter(UO));
808 return splitBinary(BO, BinarySplitter(BO));
812 std::optional<VectorSplit>
VS = getVectorSplit(GEPI.
getType());
823 for (
unsigned I = 0;
I < 1 + NumIndices; ++
I) {
826 std::optional<VectorSplit> OpVS = getVectorSplit(VecTy);
827 if (!OpVS || OpVS->NumPacked !=
VS->NumPacked) {
831 ScatterOps[
I] = scatter(&GEPI, GEPI.
getOperand(
I), *OpVS);
838 Res.resize(
VS->NumFragments);
839 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
841 SplitOps.
resize(1 + NumIndices);
842 for (
unsigned J = 0; J < 1 + NumIndices; ++J) {
844 SplitOps[J] = ScalarOps[J];
846 SplitOps[J] = ScatterOps[J][
I];
853 NewGEPI->setIsInBounds();
855 gather(&GEPI, Res, *VS);
859bool ScalarizerVisitor::visitCastInst(
CastInst &CI) {
860 std::optional<VectorSplit> DestVS = getVectorSplit(CI.
getDestTy());
864 std::optional<VectorSplit> SrcVS = getVectorSplit(CI.
getSrcTy());
865 if (!SrcVS || SrcVS->NumPacked != DestVS->NumPacked)
869 Scatterer Op0 = scatter(&CI, CI.
getOperand(0), *SrcVS);
870 assert(Op0.size() == SrcVS->NumFragments &&
"Mismatched cast");
872 Res.resize(DestVS->NumFragments);
873 for (
unsigned I = 0;
I < DestVS->NumFragments; ++
I)
877 gather(&CI, Res, *DestVS);
881bool ScalarizerVisitor::visitBitCastInst(
BitCastInst &BCI) {
882 std::optional<VectorSplit> DstVS = getVectorSplit(BCI.
getDestTy());
883 std::optional<VectorSplit> SrcVS = getVectorSplit(BCI.
getSrcTy());
884 if (!DstVS || !SrcVS || DstVS->RemainderTy || SrcVS->RemainderTy)
887 const bool isPointerTy = DstVS->VecTy->getElementType()->isPointerTy();
893 Scatterer Op0 = scatter(&BCI, BCI.
getOperand(0), *SrcVS);
895 Res.resize(DstVS->NumFragments);
897 unsigned DstSplitBits = DstVS->SplitTy->getPrimitiveSizeInBits();
898 unsigned SrcSplitBits = SrcVS->SplitTy->getPrimitiveSizeInBits();
901 assert(DstVS->NumFragments == SrcVS->NumFragments);
902 for (
unsigned I = 0;
I < DstVS->NumFragments; ++
I) {
906 }
else if (SrcSplitBits % DstSplitBits == 0) {
910 MidVS.NumPacked = DstVS->NumPacked;
911 MidVS.NumFragments = SrcSplitBits / DstSplitBits;
913 MidVS.NumPacked * MidVS.NumFragments);
914 MidVS.SplitTy = DstVS->SplitTy;
917 for (
unsigned I = 0;
I < SrcVS->NumFragments; ++
I) {
923 while ((VI = dyn_cast<Instruction>(V)) &&
924 VI->getOpcode() == Instruction::BitCast)
925 V =
VI->getOperand(0);
929 Scatterer Mid = scatter(&BCI, V, MidVS);
930 for (
unsigned J = 0; J < MidVS.NumFragments; ++J)
931 Res[ResI++] = Mid[J];
933 }
else if (DstSplitBits % SrcSplitBits == 0) {
937 MidVS.NumFragments = DstSplitBits / SrcSplitBits;
938 MidVS.NumPacked = SrcVS->NumPacked;
940 MidVS.NumPacked * MidVS.NumFragments);
941 MidVS.SplitTy = SrcVS->SplitTy;
945 ConcatOps.
resize(MidVS.NumFragments);
946 for (
unsigned I = 0;
I < DstVS->NumFragments; ++
I) {
947 for (
unsigned J = 0; J < MidVS.NumFragments; ++J)
948 ConcatOps[J] = Op0[SrcI++];
949 Value *
V = concatenate(Builder, ConcatOps, MidVS,
958 gather(&BCI, Res, *DstVS);
963 std::optional<VectorSplit>
VS = getVectorSplit(IEI.
getType());
968 Scatterer Op0 = scatter(&IEI, IEI.
getOperand(0), *VS);
973 Res.resize(
VS->NumFragments);
975 if (
auto *CI = dyn_cast<ConstantInt>(InsIdx)) {
976 unsigned Idx = CI->getZExtValue();
977 unsigned Fragment =
Idx /
VS->NumPacked;
978 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
981 if (Fragment ==
VS->NumFragments - 1 &&
VS->RemainderTy &&
982 !
VS->RemainderTy->isVectorTy())
996 if (!ScalarizeVariableInsertExtract ||
VS->NumPacked > 1)
999 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
1000 Value *ShouldReplace =
1004 Res[
I] = Builder.
CreateSelect(ShouldReplace, NewElt, OldElt,
1009 gather(&IEI, Res, *VS);
1019 Scatterer Op0 = scatter(&EEI, EEI.
getOperand(0), *VS);
1022 if (
auto *CI = dyn_cast<ConstantInt>(ExtIdx)) {
1023 unsigned Idx = CI->getZExtValue();
1024 unsigned Fragment =
Idx /
VS->NumPacked;
1025 Value *Res = Op0[Fragment];
1027 if (Fragment ==
VS->NumFragments - 1 &&
VS->RemainderTy &&
1028 !
VS->RemainderTy->isVectorTy())
1032 replaceUses(&EEI, Res);
1037 if (!ScalarizeVariableInsertExtract ||
VS->NumPacked > 1)
1041 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
1042 Value *ShouldExtract =
1049 replaceUses(&EEI, Res);
1054 std::optional<VectorSplit>
VS = getVectorSplit(SVI.
getType());
1055 std::optional<VectorSplit> VSOp =
1057 if (!VS || !VSOp ||
VS->NumPacked > 1 || VSOp->NumPacked > 1)
1060 Scatterer Op0 = scatter(&SVI, SVI.
getOperand(0), *VSOp);
1061 Scatterer Op1 = scatter(&SVI, SVI.
getOperand(1), *VSOp);
1063 Res.resize(
VS->NumFragments);
1065 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
1069 else if (
unsigned(Selector) < Op0.size())
1070 Res[
I] = Op0[Selector];
1072 Res[
I] = Op1[Selector - Op0.size()];
1074 gather(&SVI, Res, *VS);
1078bool ScalarizerVisitor::visitPHINode(
PHINode &
PHI) {
1079 std::optional<VectorSplit>
VS = getVectorSplit(
PHI.getType());
1085 Res.resize(
VS->NumFragments);
1087 unsigned NumOps =
PHI.getNumOperands();
1088 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
1093 for (
unsigned I = 0;
I < NumOps; ++
I) {
1094 Scatterer
Op = scatter(&
PHI,
PHI.getIncomingValue(
I), *VS);
1096 for (
unsigned J = 0; J <
VS->NumFragments; ++J)
1097 cast<PHINode>(Res[J])->addIncoming(
Op[J], IncomingBlock);
1099 gather(&
PHI, Res, *VS);
1103bool ScalarizerVisitor::visitLoadInst(
LoadInst &LI) {
1104 if (!ScalarizeLoadStore)
1109 std::optional<VectorLayout> Layout = getVectorLayout(
1117 Res.resize(Layout->VS.NumFragments);
1119 for (
unsigned I = 0;
I < Layout->VS.NumFragments; ++
I) {
1121 Align(Layout->getFragmentAlign(
I)),
1124 gather(&LI, Res, Layout->VS);
1128bool ScalarizerVisitor::visitStoreInst(
StoreInst &SI) {
1129 if (!ScalarizeLoadStore)
1134 Value *FullValue =
SI.getValueOperand();
1135 std::optional<VectorLayout> Layout = getVectorLayout(
1136 FullValue->
getType(),
SI.getAlign(),
SI.getDataLayout());
1141 Scatterer VPtr = scatter(&SI,
SI.getPointerOperand(), Layout->VS);
1142 Scatterer VVal = scatter(&SI, FullValue, Layout->VS);
1145 Stores.resize(Layout->VS.NumFragments);
1146 for (
unsigned I = 0;
I < Layout->VS.NumFragments; ++
I) {
1152 transferMetadataAndIRFlags(&SI, Stores);
1156bool ScalarizerVisitor::visitCallInst(
CallInst &CI) {
1157 return splitCall(CI);
1160bool ScalarizerVisitor::visitFreezeInst(
FreezeInst &FI) {
1168bool ScalarizerVisitor::finish() {
1171 if (Gathered.empty() && Scattered.empty() && !Scalarized)
1173 for (
const auto &GMI : Gathered) {
1175 ValueVector &CV = *GMI.second;
1176 if (!
Op->use_empty()) {
1180 if (
auto *Ty = dyn_cast<FixedVectorType>(
Op->getType())) {
1183 if (isa<PHINode>(
Op))
1186 VectorSplit
VS = *getVectorSplit(Ty);
1187 assert(
VS.NumFragments == CV.size());
1189 Res = concatenate(Builder, CV, VS,
Op->getName());
1193 assert(CV.size() == 1 &&
Op->getType() == CV[0]->getType());
1198 Op->replaceAllUsesWith(Res);
1200 PotentiallyDeadInstrs.emplace_back(
Op);
1213 ScalarizerVisitor Impl(DT, Options);
1214 bool Changed = Impl.visit(
F);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Module.h This file contains the declarations for the Module class.
uint64_t IntrinsicInst * II
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isTriviallyScalariable(Intrinsic::ID ID)
static cl::opt< bool > ClScalarizeVariableInsertExtract("scalarize-variable-insert-extract", cl::init(true), cl::Hidden, cl::desc("Allow the scalarizer pass to scalarize " "insertelement/extractelement with variable index"))
static cl::opt< bool > ClScalarizeLoadStore("scalarize-load-store", cl::init(false), cl::Hidden, cl::desc("Allow the scalarizer pass to scalarize loads and store"))
static cl::opt< unsigned > ClScalarizeMinBits("scalarize-min-bits", cl::init(0), cl::Hidden, cl::desc("Instruct the scalarizer pass to attempt to keep values of a " "minimum number of bits"))
This pass converts vector operations into scalar operations (or, optionally, operations on smaller ve...
This file defines the SmallVector class.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
InstListType::iterator iterator
Instruction iterators...
This class represents a no-op cast from one type to another.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Type * getSrcTy() const
Return the source type, as a convenience.
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Type * getDestTy() const
Return the destination type, as a convenience.
This is the shared class of boolean and integer constants.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
This instruction compares its operands according to the predicate given to the constructor.
Class to represent fixed width SIMD vectors.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
This class represents a freeze function that returns random concrete value if an operand is either a ...
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
bool isInBounds() const
Determine whether the GEP has the inbounds flag.
Type * getSourceElementType() const
unsigned getNumIndices() const
This instruction compares its operands according to the predicate given to the constructor.
Value * CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateFreeze(Value *V, const Twine &Name="")
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Value * CreateUnOp(Instruction::UnaryOps Opc, Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
Base class for instruction visitors.
RetTy visitFreezeInst(FreezeInst &I)
RetTy visitFCmpInst(FCmpInst &I)
RetTy visitExtractElementInst(ExtractElementInst &I)
RetTy visitShuffleVectorInst(ShuffleVectorInst &I)
RetTy visitBitCastInst(BitCastInst &I)
void visit(Iterator Start, Iterator End)
RetTy visitPHINode(PHINode &I)
RetTy visitUnaryOperator(UnaryOperator &I)
RetTy visitStoreInst(StoreInst &I)
RetTy visitInsertElementInst(InsertElementInst &I)
RetTy visitBinaryOperator(BinaryOperator &I)
RetTy visitICmpInst(ICmpInst &I)
RetTy visitCallInst(CallInst &I)
RetTy visitCastInst(CastInst &I)
RetTy visitSelectInst(SelectInst &I)
RetTy visitGetElementPtrInst(GetElementPtrInst &I)
void visitInstruction(Instruction &I)
RetTy visitLoadInst(LoadInst &I)
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
An instruction for reading from memory.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserve()
Mark an analysis as preserved.
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
This class represents the LLVM 'select' instruction.
This instruction constructs a fixed permutation of two input vectors.
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
VectorType * getType() const
Overload to return most specific vector type.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isPointerTy() const
True if this is an instance of PointerType.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
int getNumOccurrences() const
const ParentTy * getParent() const
self_iterator getIterator()
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
BasicBlock::iterator skipDebugIntrinsics(BasicBlock::iterator It)
Advance It while it points to a debug instruction and return the result.
bool isPointerTy(const Type *T)
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
DWARFExpression::Operation Op
bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(SmallVectorImpl< WeakTrackingVH > &DeadInsts, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
Same functionality as RecursivelyDeleteTriviallyDeadInstructions, but allow instructions that are not...
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
This struct is a compact representation of a valid (non-zero power of two) alignment.