34#include "llvm/IR/IntrinsicsHexagon.h"
58#define DEBUG_TYPE "hexagon-vc"
73class HexagonVectorCombine {
78 :
F(F_),
DL(
F.getDataLayout()),
AA(AA_), AC(AC_), DT(DT_),
88 Type *getByteTy(
int ElemCount = 0)
const;
91 Type *getBoolTy(
int ElemCount = 0)
const;
95 std::optional<APInt> getIntValue(
const Value *Val)
const;
101 bool isTrue(
const Value *Val)
const;
103 bool isFalse(
const Value *Val)
const;
112 int getSizeOf(
const Value *Val, SizeKind Kind = Store)
const;
113 int getSizeOf(
const Type *Ty, SizeKind Kind = Store)
const;
114 int getTypeAlignment(
Type *Ty)
const;
115 size_t length(
Value *Val)
const;
116 size_t length(
Type *Ty)
const;
125 int Length,
int Where)
const;
149 unsigned ToWidth)
const;
153 std::optional<int> calculatePointerDifference(
Value *Ptr0,
Value *Ptr1)
const;
155 unsigned getNumSignificantBits(
const Value *V,
162 template <
typename T = std::vector<Instruction *>>
165 const T &IgnoreInsts = {})
const;
168 [[maybe_unused]]
bool isByteVecTy(
Type *Ty)
const;
181 int Start,
int Length)
const;
200 AlignVectors(
const HexagonVectorCombine &HVC_) : HVC(HVC_) {}
205 using InstList = std::vector<Instruction *>;
209 AddrInfo(
const AddrInfo &) =
default;
212 : Inst(
I), Addr(
A), ValTy(
T), HaveAlign(
H),
213 NeedAlign(HVC.getTypeAlignment(ValTy)) {}
214 AddrInfo &operator=(
const AddrInfo &) =
default;
225 using AddrList = std::vector<AddrInfo>;
229 return A->comesBefore(
B);
232 using DepList = std::set<Instruction *, InstrLess>;
235 MoveGroup(
const AddrInfo &AI,
Instruction *
B,
bool Hvx,
bool Load)
236 :
Base(
B), Main{AI.Inst}, Clones{}, IsHvx(Hvx), IsLoad(Load) {}
237 MoveGroup() =
default;
245 using MoveList = std::vector<MoveGroup>;
265 Segment(
Value *Val,
int Begin,
int Len)
266 : Val(Val), Start(Begin),
Size(Len) {}
267 Segment(
const Segment &Seg) =
default;
268 Segment &operator=(
const Segment &Seg) =
default;
275 Block(
Value *Val,
int Len,
int Pos) : Seg(Val, 0, Len), Pos(Pos) {}
277 : Seg(Val, Off, Len), Pos(Pos) {}
285 ByteSpan section(
int Start,
int Length)
const;
286 ByteSpan &shift(
int Offset);
289 int size()
const {
return Blocks.size(); }
290 Block &operator[](
int i) {
return Blocks[i]; }
291 const Block &operator[](
int i)
const {
return Blocks[i]; }
293 std::vector<Block> Blocks;
296 iterator begin() {
return Blocks.begin(); }
297 iterator end() {
return Blocks.end(); }
303 std::optional<AddrInfo> getAddrInfo(
Instruction &In)
const;
304 bool isHvx(
const AddrInfo &AI)
const;
306 [[maybe_unused]]
bool isSectorTy(
Type *Ty)
const;
314 const InstMap &CloneMap = InstMap())
const;
317 const InstMap &CloneMap = InstMap())
const;
341 bool createAddressGroups();
342 MoveList createLoadGroups(
const AddrList &Group)
const;
343 MoveList createStoreGroups(
const AddrList &Group)
const;
344 bool moveTogether(MoveGroup &Move)
const;
345 template <
typename T>
348 void realignLoadGroup(
IRBuilderBase &Builder,
const ByteSpan &VSpan,
349 int ScLen,
Value *AlignVal,
Value *AlignAddr)
const;
350 void realignStoreGroup(
IRBuilderBase &Builder,
const ByteSpan &VSpan,
351 int ScLen,
Value *AlignVal,
Value *AlignAddr)
const;
352 bool realignGroup(
const MoveGroup &Move)
const;
355 int Alignment)
const;
362 std::map<Instruction *, AddrList> AddrGroups;
363 const HexagonVectorCombine &HVC;
368 OS <<
"Inst: " << AI.Inst <<
" " << *AI.Inst <<
'\n';
369 OS <<
"Addr: " << *AI.Addr <<
'\n';
370 OS <<
"Type: " << *AI.ValTy <<
'\n';
371 OS <<
"HaveAlign: " << AI.HaveAlign.
value() <<
'\n';
372 OS <<
"NeedAlign: " << AI.NeedAlign.
value() <<
'\n';
373 OS <<
"Offset: " << AI.Offset;
379 OS <<
"IsLoad:" << (MG.IsLoad ?
"yes" :
"no");
380 OS <<
", IsHvx:" << (MG.IsHvx ?
"yes" :
"no") <<
'\n';
383 OS <<
" " << *
I <<
'\n';
386 OS <<
" " << *
I <<
'\n';
388 for (
auto [K, V] : MG.Clones) {
390 K->printAsOperand(OS,
false);
391 OS <<
"\t-> " << *V <<
'\n';
398 const AlignVectors::ByteSpan::Block &
B) {
399 OS <<
" @" <<
B.Pos <<
" [" <<
B.Seg.Start <<
',' <<
B.Seg.Size <<
"] ";
400 if (
B.Seg.Val ==
reinterpret_cast<const Value *
>(&
B)) {
401 OS <<
"(self:" <<
B.Seg.Val <<
')';
402 }
else if (
B.Seg.Val !=
nullptr) {
412 OS <<
"ByteSpan[size=" << BS.size() <<
", extent=" << BS.extent() <<
'\n';
413 for (
const AlignVectors::ByteSpan::Block &
B : BS)
421 HvxIdioms(
const HexagonVectorCombine &HVC_) : HVC(HVC_) {
422 auto *
Int32Ty = HVC.getIntTy(32);
423 HvxI32Ty = HVC.getHvxTy(
Int32Ty,
false);
424 HvxP32Ty = HVC.getHvxTy(
Int32Ty,
true);
445 std::optional<unsigned> RoundAt;
450 -> std::pair<unsigned, Signedness>;
451 auto canonSgn(SValue
X, SValue
Y)
const -> std::pair<SValue, SValue>;
453 auto matchFxpMul(
Instruction &In)
const -> std::optional<FxpOp>;
457 const FxpOp &
Op)
const ->
Value *;
459 bool Rounding)
const ->
Value *;
461 bool Rounding)
const ->
Value *;
464 Value *CarryIn =
nullptr)
const
465 -> std::pair<Value *, Value *>;
470 -> std::pair<Value *, Value *>;
479 const HexagonVectorCombine &HVC;
485 const HvxIdioms::FxpOp &
Op) {
486 static const char *SgnNames[] = {
"Positive",
"Signed",
"Unsigned"};
488 if (
Op.RoundAt.has_value()) {
489 if (
Op.Frac != 0 && *
Op.RoundAt ==
Op.Frac - 1) {
492 OS <<
" + 1<<" << *
Op.RoundAt;
495 OS <<
"\n X:(" << SgnNames[
Op.X.Sgn] <<
") " << *
Op.X.Val <<
"\n"
496 <<
" Y:(" << SgnNames[
Op.Y.Sgn] <<
") " << *
Op.Y.Val;
504template <
typename T>
T *getIfUnordered(
T *MaybeT) {
505 return MaybeT && MaybeT->isUnordered() ? MaybeT :
nullptr;
517#if !defined(_MSC_VER) || _MSC_VER >= 1926
521template <
typename Pred,
typename... Ts>
522void erase_if(std::map<Ts...> &map, Pred p)
524template <
typename Pred,
typename T,
typename U>
525void erase_if(std::map<T, U> &map, Pred p)
528 for (
auto i = map.begin(), e = map.end(); i != e;) {
537template <
typename Pred,
typename T>
void erase_if(
T &&container, Pred p) {
575auto AlignVectors::ByteSpan::extent()
const ->
int {
578 int Min = Blocks[0].Pos;
579 int Max = Blocks[0].Pos + Blocks[0].Seg.Size;
580 for (
int i = 1, e =
size(); i !=
e; ++i) {
581 Min = std::min(Min, Blocks[i].Pos);
582 Max = std::max(Max, Blocks[i].Pos + Blocks[i].Seg.Size);
587auto AlignVectors::ByteSpan::section(
int Start,
int Length)
const -> ByteSpan {
589 for (
const ByteSpan::Block &
B : Blocks) {
590 int L = std::max(
B.Pos, Start);
591 int R = std::min(
B.Pos +
B.Seg.Size, Start +
Length);
594 int Off =
L >
B.Pos ?
L -
B.Pos : 0;
595 Section.Blocks.emplace_back(
B.Seg.Val,
B.Seg.Start + Off, R - L, L);
601auto AlignVectors::ByteSpan::shift(
int Offset) -> ByteSpan & {
608 SmallVector<Value *, 8> Values(Blocks.size());
609 for (
int i = 0, e = Blocks.size(); i != e; ++i)
610 Values[i] = Blocks[i].Seg.Val;
614auto AlignVectors::getAddrInfo(Instruction &In)
const
615 -> std::optional<AddrInfo> {
617 return AddrInfo(HVC, L,
L->getPointerOperand(),
L->getType(),
620 return AddrInfo(HVC, S, S->getPointerOperand(),
621 S->getValueOperand()->getType(), S->getAlign());
625 case Intrinsic::masked_load:
626 return AddrInfo(HVC,
II,
II->getArgOperand(0),
II->getType(),
627 II->getParamAlign(0).valueOrOne());
628 case Intrinsic::masked_store:
629 return AddrInfo(HVC,
II,
II->getArgOperand(1),
630 II->getArgOperand(0)->getType(),
631 II->getParamAlign(1).valueOrOne());
637auto AlignVectors::isHvx(
const AddrInfo &AI)
const ->
bool {
641auto AlignVectors::getPayload(
Value *Val)
const ->
Value * {
645 ID =
II->getIntrinsicID();
647 return In->getOperand(0);
652auto AlignVectors::getMask(
Value *Val)
const ->
Value * {
654 switch (
II->getIntrinsicID()) {
655 case Intrinsic::masked_load:
656 return II->getArgOperand(1);
657 case Intrinsic::masked_store:
658 return II->getArgOperand(2);
662 Type *ValTy = getPayload(Val)->getType();
664 return HVC.getFullValue(HVC.getBoolTy(HVC.length(VecTy)));
665 return HVC.getFullValue(HVC.getBoolTy());
668auto AlignVectors::getPassThrough(
Value *Val)
const ->
Value * {
670 if (
II->getIntrinsicID() == Intrinsic::masked_load)
671 return II->getArgOperand(2);
676auto AlignVectors::createAdjustedPointer(IRBuilderBase &Builder,
Value *
Ptr,
677 Type *ValTy,
int Adjust,
678 const InstMap &CloneMap)
const
681 if (Instruction *New = CloneMap.lookup(
I))
683 return Builder.CreatePtrAdd(
Ptr, HVC.getConstInt(Adjust),
"gep");
686auto AlignVectors::createAlignedPointer(IRBuilderBase &Builder,
Value *
Ptr,
687 Type *ValTy,
int Alignment,
688 const InstMap &CloneMap)
const
692 for (
auto [Old, New] : CloneMap)
693 I->replaceUsesOfWith(Old, New);
698 Value *AsInt = Builder.CreatePtrToInt(
Ptr, HVC.getIntTy(),
"pti");
699 Value *
Mask = HVC.getConstInt(-Alignment);
700 Value *
And = Builder.CreateAnd(remap(AsInt), Mask,
"and");
701 return Builder.CreateIntToPtr(
705auto AlignVectors::createLoad(IRBuilderBase &Builder,
Type *ValTy,
Value *
Ptr,
713 "Expectning scalar predicate");
714 if (HVC.isFalse(Predicate))
716 if (!HVC.isTrue(Predicate) && HvxHasPredLoad) {
717 Value *
Load = createPredicatedLoad(Builder, ValTy,
Ptr, Predicate,
718 Alignment, MDSources);
719 return Builder.CreateSelect(Mask, Load, PassThru);
723 assert(!HVC.isUndef(Mask));
724 if (HVC.isZero(Mask))
726 if (HVC.isTrue(Mask))
727 return createSimpleLoad(Builder, ValTy,
Ptr, Alignment, MDSources);
730 Mask, PassThru,
"mld");
735auto AlignVectors::createSimpleLoad(IRBuilderBase &Builder,
Type *ValTy,
740 Builder.CreateAlignedLoad(ValTy,
Ptr,
Align(Alignment),
"ald");
745auto AlignVectors::createPredicatedLoad(IRBuilderBase &Builder,
Type *ValTy,
751 "Predicates 'scalar' vector loads not yet supported");
753 assert(!
Predicate->getType()->isVectorTy() &&
"Expectning scalar predicate");
754 assert(HVC.getSizeOf(ValTy, HVC.Alloc) % Alignment == 0);
755 if (HVC.isFalse(Predicate))
757 if (HVC.isTrue(Predicate))
758 return createSimpleLoad(Builder, ValTy,
Ptr, Alignment, MDSources);
760 auto V6_vL32b_pred_ai = HVC.HST.
getIntrinsicId(Hexagon::V6_vL32b_pred_ai);
762 return HVC.createHvxIntrinsic(Builder, V6_vL32b_pred_ai, ValTy,
767auto AlignVectors::createStore(IRBuilderBase &Builder,
Value *Val,
Value *
Ptr,
770 if (HVC.isZero(Mask) || HVC.isUndef(Val) || HVC.isUndef(Mask))
773 "Expectning scalar predicate"));
775 if (HVC.isFalse(Predicate))
777 if (HVC.isTrue(Predicate))
782 if (HVC.isTrue(Mask)) {
784 return createPredicatedStore(Builder, Val,
Ptr, Predicate, Alignment,
788 return createSimpleStore(Builder, Val,
Ptr, Alignment, MDSources);
794 Builder.CreateMaskedStore(Val,
Ptr,
Align(Alignment), Mask);
801 Value *PredLoad = createPredicatedLoad(Builder, Val->getType(),
Ptr,
802 Predicate, Alignment, MDSources);
803 Value *Mux = Builder.CreateSelect(Mask, Val, PredLoad);
804 return createPredicatedStore(Builder, Mux,
Ptr, Predicate, Alignment,
808auto AlignVectors::createSimpleStore(IRBuilderBase &Builder,
Value *Val,
817auto AlignVectors::createPredicatedStore(IRBuilderBase &Builder,
Value *Val,
823 "Predicates 'scalar' vector stores not yet supported");
825 if (HVC.isFalse(Predicate))
827 if (HVC.isTrue(Predicate))
828 return createSimpleStore(Builder, Val,
Ptr, Alignment, MDSources);
830 assert(HVC.getSizeOf(Val, HVC.Alloc) % Alignment == 0);
831 auto V6_vS32b_pred_ai = HVC.HST.
getIntrinsicId(Hexagon::V6_vS32b_pred_ai);
833 return HVC.createHvxIntrinsic(Builder, V6_vS32b_pred_ai,
nullptr,
838auto AlignVectors::getUpwardDeps(Instruction *In, Instruction *
Base)
const
842 "Base and In should be in the same block");
843 assert(
Base->comesBefore(In) &&
"Base should come before In");
846 std::deque<Instruction *> WorkQ = {
In};
847 while (!WorkQ.empty()) {
854 if (
I->getParent() == Parent &&
Base->comesBefore(
I))
862auto AlignVectors::createAddressGroups() ->
bool {
867 auto findBaseAndOffset = [&](AddrInfo &AI) -> std::pair<Instruction *, int> {
868 for (AddrInfo &W : WorkStack) {
869 if (
auto D = HVC.calculatePointerDifference(AI.Addr,
W.Addr))
870 return std::make_pair(
W.Inst, *
D);
872 return std::make_pair(
nullptr, 0);
875 auto traverseBlock = [&](
DomTreeNode *DomN,
auto Visit) ->
void {
877 for (Instruction &
I :
Block) {
878 auto AI = this->getAddrInfo(
I);
881 auto F = findBaseAndOffset(*AI);
883 if (Instruction *BI =
F.first) {
884 AI->Offset =
F.second;
887 WorkStack.push_back(*AI);
888 GroupInst = AI->Inst;
890 AddrGroups[GroupInst].push_back(*AI);
896 while (!WorkStack.empty() && WorkStack.back().Inst->getParent() == &
Block)
897 WorkStack.pop_back();
900 traverseBlock(HVC.DT.
getRootNode(), traverseBlock);
901 assert(WorkStack.empty());
906 erase_if(AddrGroups, [](
auto &
G) {
return G.second.size() == 1; });
910 G.second, [&](
auto &
I) { return HVC.HST.isTypeForHVX(I.ValTy); });
913 return !AddrGroups.empty();
916auto AlignVectors::createLoadGroups(
const AddrList &Group)
const -> MoveList {
924 auto tryAddTo = [&](
const AddrInfo &
Info, MoveGroup &Move) {
925 assert(!Move.Main.empty() &&
"Move group should have non-empty Main");
929 if (Move.IsHvx != isHvx(
Info))
933 if (
Base->getParent() !=
Info.Inst->getParent())
936 if (!HVC.isSafeToMoveBeforeInBB(*
Info.Inst,
Base->getIterator()))
940 return HVC.isSafeToMoveBeforeInBB(*
I,
Base->getIterator()) &&
941 HVC.isSafeToClone(*
I);
943 DepList Deps = getUpwardDeps(
Info.Inst,
Base);
947 Move.Main.push_back(
Info.Inst);
954 for (
const AddrInfo &
Info : Group) {
955 if (!
Info.Inst->mayReadFromMemory())
957 if (LoadGroups.empty() || !tryAddTo(
Info, LoadGroups.back()))
958 LoadGroups.emplace_back(
Info, Group.front().Inst, isHvx(
Info),
true);
962 erase_if(LoadGroups, [](
const MoveGroup &
G) {
return G.Main.size() <= 1; });
966 erase_if(LoadGroups, [](
const MoveGroup &
G) {
return G.IsHvx; });
971auto AlignVectors::createStoreGroups(
const AddrList &Group)
const -> MoveList {
979 auto tryAddTo = [&](
const AddrInfo &
Info, MoveGroup &Move) {
980 assert(!Move.Main.empty() &&
"Move group should have non-empty Main");
986 "Not handling stores with return values");
988 if (Move.IsHvx != isHvx(
Info))
994 if (
Base->getParent() !=
Info.Inst->getParent())
996 if (!HVC.isSafeToMoveBeforeInBB(*
Info.Inst,
Base->getIterator(), Move.Main))
998 Move.Main.push_back(
Info.Inst);
1002 MoveList StoreGroups;
1004 for (
auto I = Group.rbegin(),
E = Group.rend();
I !=
E; ++
I) {
1005 const AddrInfo &
Info = *
I;
1006 if (!
Info.Inst->mayWriteToMemory())
1008 if (StoreGroups.empty() || !tryAddTo(
Info, StoreGroups.back()))
1009 StoreGroups.emplace_back(
Info, Group.front().Inst, isHvx(
Info),
false);
1013 erase_if(StoreGroups, [](
const MoveGroup &
G) {
return G.Main.size() <= 1; });
1017 erase_if(StoreGroups, [](
const MoveGroup &
G) {
return G.IsHvx; });
1022 if (!VADoFullStores) {
1023 erase_if(StoreGroups, [
this](
const MoveGroup &
G) {
1025 auto MaybeInfo = this->getAddrInfo(*S);
1026 assert(MaybeInfo.has_value());
1027 return HVC.HST.isHVXVectorType(
1028 EVT::getEVT(MaybeInfo->ValTy, false));
1036auto AlignVectors::moveTogether(MoveGroup &Move)
const ->
bool {
1038 assert(!Move.Main.empty() &&
"Move group should have non-empty Main");
1044 Move.Clones = cloneBefore(Where->
getIterator(), Move.Deps);
1047 for (Instruction *M : Main) {
1049 M->moveAfter(Where);
1050 for (
auto [Old, New] : Move.Clones)
1051 M->replaceUsesOfWith(Old, New);
1055 for (
int i = 0, e = Move.Deps.size(); i != e; ++i)
1056 Move.Deps[i] = Move.Clones[Move.Deps[i]];
1061 assert(Move.Deps.empty());
1064 for (Instruction *M : Main.drop_front(1)) {
1070 return Move.Main.size() + Move.Deps.size() > 1;
1073template <
typename T>
1078 for (Instruction *
I : Insts) {
1079 assert(HVC.isSafeToClone(*
I));
1081 C->setName(Twine(
"c.") +
I->getName() +
".");
1082 C->insertBefore(To);
1084 for (
auto [Old, New] : Map)
1085 C->replaceUsesOfWith(Old, New);
1086 Map.insert(std::make_pair(
I,
C));
1091auto AlignVectors::realignLoadGroup(IRBuilderBase &Builder,
1092 const ByteSpan &VSpan,
int ScLen,
1097 Type *SecTy = HVC.getByteTy(ScLen);
1098 int NumSectors = (VSpan.extent() + ScLen - 1) / ScLen;
1099 bool DoAlign = !HVC.isZero(AlignVal);
1101 BasicBlock *BaseBlock = Builder.GetInsertBlock();
1104 auto *True = HVC.getFullValue(HVC.getBoolTy(ScLen));
1131 for (
int Index = 0;
Index != NumSectors; ++
Index)
1132 ASpan.Blocks.emplace_back(
nullptr, ScLen, Index * ScLen);
1133 for (
int Index = 0;
Index != NumSectors; ++
Index) {
1134 ASpan.Blocks[
Index].Seg.Val =
1135 reinterpret_cast<Value *
>(&ASpan.Blocks[
Index]);
1141 DenseMap<void *, Instruction *> EarliestUser;
1147 assert(
A->getParent() ==
B->getParent());
1148 return A->comesBefore(
B);
1150 auto earliestUser = [&](
const auto &
Uses) {
1152 for (
const Use &U :
Uses) {
1154 assert(
I !=
nullptr &&
"Load used in a non-instruction?");
1158 if (
I->getParent() == BaseBlock) {
1160 User = std::min(User,
I, isEarlier);
1168 for (
const ByteSpan::Block &
B : VSpan) {
1169 ByteSpan ASection = ASpan.section(
B.Pos,
B.Seg.Size);
1170 for (
const ByteSpan::Block &S : ASection) {
1171 auto &EU = EarliestUser[S.Seg.Val];
1172 EU = std::min(EU, earliestUser(
B.Seg.Val->uses()), isEarlier);
1177 dbgs() <<
"ASpan:\n" << ASpan <<
'\n';
1178 dbgs() <<
"Earliest users of ASpan:\n";
1179 for (
auto &[Val, User] : EarliestUser) {
1180 dbgs() << Val <<
"\n ->" << *
User <<
'\n';
1184 auto createLoad = [&](IRBuilderBase &Builder,
const ByteSpan &VSpan,
1185 int Index,
bool MakePred) {
1187 createAdjustedPointer(Builder, AlignAddr, SecTy, Index * ScLen);
1189 MakePred ? makeTestIfUnaligned(Builder, AlignVal, ScLen) : nullptr;
1194 int Width = (1 + DoAlign) * ScLen;
1195 return this->createLoad(Builder, SecTy,
Ptr, Predicate, ScLen, True, Undef,
1196 VSpan.section(Start, Width).values());
1201 assert(
In->getParent() == To->getParent());
1202 DepList Deps = getUpwardDeps(&*In, &*To);
1205 InstMap
Map = cloneBefore(In, Deps);
1206 for (
auto [Old, New] : Map)
1207 In->replaceUsesOfWith(Old, New);
1212 for (
int Index = 0;
Index != NumSectors + 1; ++
Index) {
1220 DoAlign &&
Index > 0 ? EarliestUser[&ASpan[
Index - 1]] :
nullptr;
1222 Index < NumSectors ? EarliestUser[&ASpan[
Index]] :
nullptr;
1223 if (
auto *Where = std::min(PrevAt, ThisAt, isEarlier)) {
1226 createLoad(Builder, VSpan, Index, DoAlign && Index == NumSectors);
1234 if (!HVC.isSafeToMoveBeforeInBB(*Load, BasePos))
1235 moveBefore(
Load->getIterator(), BasePos);
1237 LLVM_DEBUG(
dbgs() <<
"Loads[" << Index <<
"]:" << *Loads[Index] <<
'\n');
1243 for (
int Index = 0;
Index != NumSectors; ++
Index) {
1244 ASpan[
Index].Seg.Val =
nullptr;
1245 if (
auto *Where = EarliestUser[&ASpan[Index]]) {
1251 assert(NextLoad !=
nullptr);
1252 Val = HVC.vralignb(Builder, Val, NextLoad, AlignVal);
1254 ASpan[
Index].Seg.Val = Val;
1259 for (
const ByteSpan::Block &
B : VSpan) {
1260 ByteSpan ASection = ASpan.section(
B.Pos,
B.Seg.Size).shift(-
B.Pos);
1267 std::vector<ByteSpan::Block *> ABlocks;
1268 for (ByteSpan::Block &S : ASection) {
1269 if (S.Seg.Val !=
nullptr)
1270 ABlocks.push_back(&S);
1273 [&](
const ByteSpan::Block *
A,
const ByteSpan::Block *
B) {
1277 for (ByteSpan::Block *S : ABlocks) {
1282 Value *Pay = HVC.vbytes(Builder, getPayload(S->Seg.Val));
1284 HVC.insertb(Builder, Accum, Pay, S->Seg.Start, S->Seg.Size, S->Pos);
1292 Type *ValTy = getPayload(
B.Seg.Val)->getType();
1295 getPassThrough(
B.Seg.Val),
"sel");
1300auto AlignVectors::realignStoreGroup(IRBuilderBase &Builder,
1301 const ByteSpan &VSpan,
int ScLen,
1306 Type *SecTy = HVC.getByteTy(ScLen);
1307 int NumSectors = (VSpan.extent() + ScLen - 1) / ScLen;
1308 bool DoAlign = !HVC.isZero(AlignVal);
1311 ByteSpan ASpanV, ASpanM;
1315 auto MakeVec = [](IRBuilderBase &Builder,
Value *Val) ->
Value * {
1319 auto *VecTy = VectorType::get(Ty, 1,
false);
1325 for (
int Index = (DoAlign ? -1 : 0);
Index != NumSectors + DoAlign; ++
Index) {
1329 VSpan.section(Index * ScLen, ScLen).shift(-Index * ScLen);
1334 for (ByteSpan::Block &S : VSection) {
1335 Value *Pay = getPayload(S.Seg.Val);
1337 Pay->
getType(), HVC.getByteTy());
1338 Value *PartM = HVC.insertb(Builder, Zero, HVC.vbytes(Builder, Mask),
1339 S.Seg.Start, S.Seg.Size, S.Pos);
1340 AccumM = Builder.
CreateOr(AccumM, PartM);
1342 Value *PartV = HVC.insertb(Builder, Undef, HVC.vbytes(Builder, Pay),
1343 S.Seg.Start, S.Seg.Size, S.Pos);
1348 ASpanV.Blocks.emplace_back(AccumV, ScLen, Index * ScLen);
1349 ASpanM.Blocks.emplace_back(AccumM, ScLen, Index * ScLen);
1353 dbgs() <<
"ASpanV before vlalign:\n" << ASpanV <<
'\n';
1354 dbgs() <<
"ASpanM before vlalign:\n" << ASpanM <<
'\n';
1359 for (
int Index = 1;
Index != NumSectors + 2; ++
Index) {
1360 Value *PrevV = ASpanV[
Index - 1].Seg.Val, *ThisV = ASpanV[
Index].Seg.Val;
1361 Value *PrevM = ASpanM[
Index - 1].Seg.Val, *ThisM = ASpanM[
Index].Seg.Val;
1363 ASpanV[
Index - 1].Seg.Val = HVC.vlalignb(Builder, PrevV, ThisV, AlignVal);
1364 ASpanM[
Index - 1].Seg.Val = HVC.vlalignb(Builder, PrevM, ThisM, AlignVal);
1369 dbgs() <<
"ASpanV after vlalign:\n" << ASpanV <<
'\n';
1370 dbgs() <<
"ASpanM after vlalign:\n" << ASpanM <<
'\n';
1373 auto createStore = [&](IRBuilderBase &Builder,
const ByteSpan &ASpanV,
1374 const ByteSpan &ASpanM,
int Index,
bool MakePred) {
1377 if (HVC.isUndef(Val) || HVC.isZero(Mask))
1380 createAdjustedPointer(Builder, AlignAddr, SecTy, Index * ScLen);
1382 MakePred ? makeTestIfUnaligned(Builder, AlignVal, ScLen) : nullptr;
1387 int Width = (1 + DoAlign) * ScLen;
1388 this->createStore(Builder, Val,
Ptr, Predicate, ScLen,
1389 HVC.vlsb(Builder, Mask),
1390 VSpan.section(Start, Width).values());
1393 for (
int Index = 0;
Index != NumSectors + DoAlign; ++
Index) {
1394 createStore(Builder, ASpanV, ASpanM, Index, DoAlign && Index == NumSectors);
1398auto AlignVectors::realignGroup(
const MoveGroup &Move)
const ->
bool {
1407 auto getMaxOf = [](
auto Range,
auto GetValue) {
1409 return GetValue(
A) < GetValue(
B);
1413 const AddrList &BaseInfos = AddrGroups.at(Move.Base);
1428 std::set<Instruction *> TestSet(Move.Main.begin(), Move.Main.end());
1431 BaseInfos, std::back_inserter(MoveInfos),
1432 [&TestSet](
const AddrInfo &AI) {
return TestSet.count(AI.Inst); });
1435 const AddrInfo &WithMaxAlign =
1436 getMaxOf(MoveInfos, [](
const AddrInfo &AI) {
return AI.HaveAlign; });
1437 Align MaxGiven = WithMaxAlign.HaveAlign;
1440 const AddrInfo &WithMinOffset =
1441 getMaxOf(MoveInfos, [](
const AddrInfo &AI) {
return -AI.Offset; });
1443 const AddrInfo &WithMaxNeeded =
1444 getMaxOf(MoveInfos, [](
const AddrInfo &AI) {
return AI.NeedAlign; });
1445 Align MinNeeded = WithMaxNeeded.NeedAlign;
1458 InstSimplifyFolder(HVC.DL));
1459 Value *AlignAddr =
nullptr;
1460 Value *AlignVal =
nullptr;
1462 if (MinNeeded <= MaxGiven) {
1463 int Start = WithMinOffset.Offset;
1464 int OffAtMax = WithMaxAlign.Offset;
1471 int Adjust = -
alignTo(OffAtMax - Start, MinNeeded.value());
1472 AlignAddr = createAdjustedPointer(Builder, WithMaxAlign.Addr,
1473 WithMaxAlign.ValTy, Adjust, Move.Clones);
1474 int Diff =
Start - (OffAtMax + Adjust);
1475 AlignVal = HVC.getConstInt(Diff);
1477 assert(
static_cast<decltype(MinNeeded.value())
>(Diff) < MinNeeded.value());
1487 createAlignedPointer(Builder, WithMinOffset.Addr, WithMinOffset.ValTy,
1488 MinNeeded.value(), Move.Clones);
1490 Builder.
CreatePtrToInt(WithMinOffset.Addr, HVC.getIntTy(),
"pti");
1492 for (
auto [Old, New] : Move.Clones)
1493 I->replaceUsesOfWith(Old, New);
1498 for (
const AddrInfo &AI : MoveInfos) {
1499 VSpan.Blocks.emplace_back(AI.Inst, HVC.getSizeOf(AI.ValTy),
1500 AI.Offset - WithMinOffset.Offset);
1507 : std::max<int>(MinNeeded.value(), 4);
1508 assert(!Move.IsHvx || ScLen == 64 || ScLen == 128);
1509 assert(Move.IsHvx || ScLen == 4 || ScLen == 8);
1512 dbgs() <<
"ScLen: " << ScLen <<
"\n";
1513 dbgs() <<
"AlignVal:" << *AlignVal <<
"\n";
1514 dbgs() <<
"AlignAddr:" << *AlignAddr <<
"\n";
1515 dbgs() <<
"VSpan:\n" << VSpan <<
'\n';
1519 realignLoadGroup(Builder, VSpan, ScLen, AlignVal, AlignAddr);
1521 realignStoreGroup(Builder, VSpan, ScLen, AlignVal, AlignAddr);
1523 for (
auto *Inst : Move.Main)
1524 Inst->eraseFromParent();
1529auto AlignVectors::makeTestIfUnaligned(IRBuilderBase &Builder,
Value *AlignVal,
1530 int Alignment)
const ->
Value * {
1531 auto *AlignTy = AlignVal->getType();
1533 AlignVal, ConstantInt::get(AlignTy, Alignment - 1),
"and");
1534 Value *
Zero = ConstantInt::get(AlignTy, 0);
1538auto AlignVectors::isSectorTy(
Type *Ty)
const ->
bool {
1539 if (!HVC.isByteVecTy(Ty))
1541 int Size = HVC.getSizeOf(Ty);
1547auto AlignVectors::run() ->
bool {
1550 if (!createAddressGroups())
1554 dbgs() <<
"Address groups(" << AddrGroups.size() <<
"):\n";
1555 for (
auto &[In, AL] : AddrGroups) {
1556 for (
const AddrInfo &AI : AL)
1557 dbgs() <<
"---\n" << AI <<
'\n';
1562 MoveList LoadGroups, StoreGroups;
1564 for (
auto &
G : AddrGroups) {
1570 dbgs() <<
"\nLoad groups(" << LoadGroups.size() <<
"):\n";
1571 for (
const MoveGroup &
G : LoadGroups)
1572 dbgs() <<
G <<
"\n";
1573 dbgs() <<
"Store groups(" << StoreGroups.size() <<
"):\n";
1574 for (
const MoveGroup &
G : StoreGroups)
1575 dbgs() <<
G <<
"\n";
1579 unsigned CountLimit = VAGroupCountLimit;
1580 if (CountLimit == 0)
1583 if (LoadGroups.size() > CountLimit) {
1584 LoadGroups.resize(CountLimit);
1585 StoreGroups.clear();
1587 unsigned StoreLimit = CountLimit - LoadGroups.size();
1588 if (StoreGroups.size() > StoreLimit)
1589 StoreGroups.resize(StoreLimit);
1592 for (
auto &M : LoadGroups)
1594 for (
auto &M : StoreGroups)
1599 for (
auto &M : LoadGroups)
1601 for (
auto &M : StoreGroups)
1611auto HvxIdioms::getNumSignificantBits(
Value *V, Instruction *In)
const
1612 -> std::pair<unsigned, Signedness> {
1613 unsigned Bits = HVC.getNumSignificantBits(V, In);
1619 KnownBits Known = HVC.getKnownBits(V, In);
1620 Signedness Sign =
Signed;
1621 unsigned NumToTest = 0;
1625 NumToTest =
Bits - 1;
1638 return {
Bits, Sign};
1641auto HvxIdioms::canonSgn(SValue
X, SValue
Y)
const
1642 -> std::pair<SValue, SValue> {
1655auto HvxIdioms::matchFxpMul(Instruction &In)
const -> std::optional<FxpOp> {
1656 using namespace PatternMatch;
1657 auto *Ty =
In.getType();
1660 return std::nullopt;
1669 auto m_Shr = [](
auto &&
V,
auto &&S) {
1681 if (
Op.Frac > Width)
1682 return std::nullopt;
1689 return std::nullopt;
1697 Op.Opcode = Instruction::Mul;
1699 Op.X.Sgn = getNumSignificantBits(
Op.X.Val, &In).second;
1700 Op.Y.Sgn = getNumSignificantBits(
Op.Y.Val, &In).second;
1705 return std::nullopt;
1708auto HvxIdioms::processFxpMul(Instruction &In,
const FxpOp &
Op)
const
1710 assert(
Op.X.Val->getType() ==
Op.Y.Val->getType());
1713 if (VecTy ==
nullptr)
1716 unsigned ElemWidth = ElemTy->getBitWidth();
1719 if ((HVC.length(VecTy) * ElemWidth) % (8 * HVC.HST.
getVectorLength()) != 0)
1729 if (ElemWidth <= 32 &&
Op.Frac == 0)
1732 auto [BitsX, SignX] = getNumSignificantBits(
Op.X.Val, &In);
1733 auto [BitsY, SignY] = getNumSignificantBits(
Op.Y.Val, &In);
1739 InstSimplifyFolder(HVC.DL));
1741 auto roundUpWidth = [](
unsigned Width) ->
unsigned {
1747 if (Width > 32 && Width % 32 != 0) {
1754 BitsX = roundUpWidth(BitsX);
1755 BitsY = roundUpWidth(BitsY);
1760 unsigned Width = std::max(BitsX, BitsY);
1762 auto *ResizeTy = VectorType::get(HVC.getIntTy(Width), VecTy);
1763 if (Width < ElemWidth) {
1766 }
else if (Width > ElemWidth) {
1773 assert(
X->getType() ==
Y->getType() &&
X->getType() == ResizeTy);
1775 unsigned VecLen = HVC.length(ResizeTy);
1776 unsigned ChopLen = (8 * HVC.HST.
getVectorLength()) / std::min(Width, 32u);
1780 ChopOp.ResTy = VectorType::get(
Op.ResTy->getElementType(), ChopLen,
false);
1782 for (
unsigned V = 0;
V != VecLen / ChopLen; ++
V) {
1783 ChopOp.X.Val = HVC.subvector(Builder,
X, V * ChopLen, ChopLen);
1784 ChopOp.Y.Val = HVC.subvector(Builder,
Y, V * ChopLen, ChopLen);
1785 Results.push_back(processFxpMulChopped(Builder, In, ChopOp));
1800auto HvxIdioms::processFxpMulChopped(IRBuilderBase &Builder, Instruction &In,
1801 const FxpOp &
Op)
const ->
Value * {
1802 assert(
Op.X.Val->getType() ==
Op.Y.Val->getType());
1804 unsigned Width = InpTy->getScalarSizeInBits();
1807 if (!
Op.RoundAt || *
Op.RoundAt ==
Op.Frac - 1) {
1810 Value *QMul =
nullptr;
1812 QMul = createMulQ15(Builder,
Op.X,
Op.Y, Rounding);
1813 }
else if (Width == 32) {
1814 QMul = createMulQ31(Builder,
Op.X,
Op.Y, Rounding);
1816 if (QMul !=
nullptr)
1822 assert(Width < 32 || Width % 32 == 0);
1832 assert(
Op.Frac != 0 &&
"Unshifted mul should have been skipped");
1833 if (
Op.Frac == 16) {
1835 if (
Value *MulH = createMulH16(Builder,
Op.X,
Op.Y))
1839 Value *Prod32 = createMul16(Builder,
Op.X,
Op.Y);
1841 Value *RoundVal = HVC.getConstSplat(Prod32->
getType(), 1 << *
Op.RoundAt);
1842 Prod32 = Builder.
CreateAdd(Prod32, RoundVal,
"add");
1847 ? Builder.
CreateAShr(Prod32, ShiftAmt,
"asr")
1848 : Builder.
CreateLShr(Prod32, ShiftAmt,
"lsr");
1849 return Builder.
CreateTrunc(Shifted, InpTy,
"trn");
1856 auto WordX = HVC.splitVectorElements(Builder,
Op.X.Val, 32);
1857 auto WordY = HVC.splitVectorElements(Builder,
Op.Y.Val, 32);
1858 auto WordP = createMulLong(Builder, WordX,
Op.X.Sgn, WordY,
Op.Y.Sgn);
1863 if (
Op.RoundAt.has_value()) {
1866 RoundV[*
Op.RoundAt / 32] =
1867 HVC.getConstSplat(HvxWordTy, 1 << (*
Op.RoundAt % 32));
1868 WordP = createAddLong(Builder, WordP, RoundV);
1874 unsigned SkipWords =
Op.Frac / 32;
1875 Constant *ShiftAmt = HVC.getConstSplat(HvxWordTy,
Op.Frac % 32);
1877 for (
int Dst = 0, End = WordP.size() - SkipWords; Dst != End; ++Dst) {
1878 int Src = Dst + SkipWords;
1880 if (Src + 1 < End) {
1891 WordP.resize(WordP.size() - SkipWords);
1893 return HVC.joinVectorElements(Builder, WordP,
Op.ResTy);
1896auto HvxIdioms::createMulQ15(IRBuilderBase &Builder, SValue
X, SValue
Y,
1897 bool Rounding)
const ->
Value * {
1898 assert(
X.Val->getType() ==
Y.Val->getType());
1899 assert(
X.Val->getType()->getScalarType() == HVC.getIntTy(16));
1906 auto V6_vmpyhvsrs = HVC.HST.
getIntrinsicId(Hexagon::V6_vmpyhvsrs);
1907 return HVC.createHvxIntrinsic(Builder, V6_vmpyhvsrs,
X.Val->getType(),
1911auto HvxIdioms::createMulQ31(IRBuilderBase &Builder, SValue
X, SValue
Y,
1912 bool Rounding)
const ->
Value * {
1913 Type *InpTy =
X.Val->getType();
1914 assert(InpTy ==
Y.Val->getType());
1926 HVC.createHvxIntrinsic(Builder, V6_vmpyewuh, InpTy, {
X.Val,
Y.Val});
1927 return HVC.createHvxIntrinsic(Builder, V6_vmpyo_acc, InpTy,
1928 {V1,
X.Val,
Y.Val});
1931auto HvxIdioms::createAddCarry(IRBuilderBase &Builder,
Value *
X,
Value *
Y,
1932 Value *CarryIn)
const
1933 -> std::pair<Value *, Value *> {
1934 assert(
X->getType() ==
Y->getType());
1943 if (CarryIn ==
nullptr)
1944 CarryIn = HVC.getNullValue(HVC.getBoolTy(HVC.length(VecTy)));
1945 Args.push_back(CarryIn);
1947 Value *
Ret = HVC.createHvxIntrinsic(Builder, AddCarry,
1951 return {
Result, CarryOut};
1958 if (CarryIn !=
nullptr) {
1959 unsigned Width = VecTy->getScalarSizeInBits();
1962 for (
unsigned i = 0, e = 32 / Width; i !=
e; ++i)
1963 Mask = (Mask << Width) | 1;
1967 HVC.createHvxIntrinsic(Builder, V6_vandqrt,
nullptr,
1968 {CarryIn, HVC.getConstInt(Mask)});
1969 Result1 = Builder.
CreateAdd(
X, ValueIn,
"add");
1975 return {Result2, Builder.
CreateOr(CarryOut1, CarryOut2,
"orb")};
1978auto HvxIdioms::createMul16(IRBuilderBase &Builder, SValue
X, SValue
Y)
const
1981 std::tie(
X,
Y) = canonSgn(
X,
Y);
1994 HVC.createHvxIntrinsic(Builder, V6_vmpyh, HvxP32Ty, {
Y.Val,
X.Val});
1996 return HVC.vshuff(Builder, HVC.sublo(Builder,
P), HVC.subhi(Builder,
P));
1999auto HvxIdioms::createMulH16(IRBuilderBase &Builder, SValue
X, SValue
Y)
const
2001 Type *HvxI16Ty = HVC.getHvxTy(HVC.getIntTy(16),
false);
2006 return HVC.createHvxIntrinsic(Builder, V6_vmpyuhvs, HvxI16Ty,
2011 Type *HvxP16Ty = HVC.getHvxTy(HVC.getIntTy(16),
true);
2014 unsigned Len = HVC.length(HvxP16Ty) / 2;
2016 SmallVector<int, 128> PickOdd(Len);
2017 for (
int i = 0; i !=
static_cast<int>(
Len); ++i)
2018 PickOdd[i] = 2 * i + 1;
2021 HVC.sublo(Builder, Pair16), HVC.subhi(Builder, Pair16), PickOdd,
"shf");
2024auto HvxIdioms::createMul32(IRBuilderBase &Builder, SValue
X, SValue
Y)
const
2025 -> std::pair<Value *, Value *> {
2026 assert(
X.Val->getType() ==
Y.Val->getType());
2027 assert(
X.Val->getType() == HvxI32Ty);
2030 std::tie(
X,
Y) = canonSgn(
X,
Y);
2033 V6_vmpy_parts = Intrinsic::hexagon_V6_vmpyss_parts;
2035 V6_vmpy_parts = Intrinsic::hexagon_V6_vmpyus_parts;
2037 V6_vmpy_parts = Intrinsic::hexagon_V6_vmpyuu_parts;
2040 Value *Parts = HVC.createHvxIntrinsic(Builder, V6_vmpy_parts,
nullptr,
2041 {
X.Val,
Y.Val}, {HvxI32Ty});
2050 assert(WordX.size() == WordY.size());
2051 unsigned Idx = 0,
Length = WordX.size();
2055 if (HVC.isZero(WordX[Idx]))
2056 Sum[Idx] = WordY[Idx];
2057 else if (HVC.isZero(WordY[Idx]))
2058 Sum[Idx] = WordX[Idx];
2064 Value *Carry =
nullptr;
2065 for (; Idx !=
Length; ++Idx) {
2066 std::tie(Sum[Idx], Carry) =
2067 createAddCarry(Builder, WordX[Idx], WordY[Idx], Carry);
2081 for (
int i = 0, e = WordX.size(); i != e; ++i) {
2082 for (
int j = 0, f = WordY.size(); j != f; ++j) {
2084 Signedness SX = (i + 1 ==
e) ? SgnX :
Unsigned;
2086 auto [
Lo,
Hi] = createMul32(Builder, {WordX[i], SX}, {WordY[
j],
SY});
2087 Products[i +
j + 0].push_back(
Lo);
2088 Products[i +
j + 1].push_back(
Hi);
2102 for (
int i = 0, e = Products.size(); i != e; ++i) {
2103 while (Products[i].
size() > 1) {
2104 Value *Carry =
nullptr;
2105 for (
int j = i;
j !=
e; ++
j) {
2106 auto &ProdJ = Products[
j];
2107 auto [Sum, CarryOut] = createAddCarry(Builder, pop_back_or_zero(ProdJ),
2108 pop_back_or_zero(ProdJ), Carry);
2109 ProdJ.insert(ProdJ.begin(), Sum);
2116 for (
auto &
P : Products) {
2117 assert(
P.size() == 1 &&
"Should have been added together");
2124auto HvxIdioms::run() ->
bool {
2127 for (BasicBlock &
B : HVC.F) {
2128 for (
auto It =
B.rbegin(); It !=
B.rend(); ++It) {
2129 if (
auto Fxm = matchFxpMul(*It)) {
2130 Value *
New = processFxpMul(*It, *Fxm);
2136 It->replaceAllUsesWith(New);
2138 It = StartOver ?
B.rbegin()
2150auto HexagonVectorCombine::run() ->
bool {
2152 dbgs() <<
"Module before HexagonVectorCombine\n" << *
F.getParent();
2155 if (HST.useHVXOps()) {
2157 Changed |= AlignVectors(*this).run();
2159 Changed |= HvxIdioms(*this).run();
2163 dbgs() <<
"Module " << (
Changed ?
"(modified)" :
"(unchanged)")
2164 <<
" after HexagonVectorCombine\n"
2170auto HexagonVectorCombine::getIntTy(
unsigned Width)
const -> IntegerType * {
2174auto HexagonVectorCombine::getByteTy(
int ElemCount)
const ->
Type * {
2176 IntegerType *ByteTy = Type::getInt8Ty(
F.getContext());
2179 return VectorType::get(ByteTy, ElemCount,
false);
2182auto HexagonVectorCombine::getBoolTy(
int ElemCount)
const ->
Type * {
2184 IntegerType *BoolTy = Type::getInt1Ty(
F.getContext());
2187 return VectorType::get(BoolTy, ElemCount,
false);
2190auto HexagonVectorCombine::getConstInt(
int Val,
unsigned Width)
const
2195auto HexagonVectorCombine::isZero(
const Value *Val)
const ->
bool {
2197 return C->isZeroValue();
2201auto HexagonVectorCombine::getIntValue(
const Value *Val)
const
2202 -> std::optional<APInt> {
2204 return CI->getValue();
2205 return std::nullopt;
2208auto HexagonVectorCombine::isUndef(
const Value *Val)
const ->
bool {
2212auto HexagonVectorCombine::isTrue(
const Value *Val)
const ->
bool {
2216auto HexagonVectorCombine::isFalse(
const Value *Val)
const ->
bool {
2220auto HexagonVectorCombine::getHvxTy(
Type *ElemTy,
bool Pair)
const
2226 "Invalid HVX element type");
2227 unsigned HwLen = HST.getVectorLength();
2229 return VectorType::get(ElemTy, Pair ? 2 * NumElems : NumElems,
2233auto HexagonVectorCombine::getSizeOf(
const Value *Val, SizeKind Kind)
const
2235 return getSizeOf(Val->
getType(), Kind);
2238auto HexagonVectorCombine::getSizeOf(
const Type *Ty, SizeKind Kind)
const
2240 auto *NcTy =
const_cast<Type *
>(Ty);
2243 return DL.getTypeStoreSize(NcTy).getFixedValue();
2245 return DL.getTypeAllocSize(NcTy).getFixedValue();
2250auto HexagonVectorCombine::getTypeAlignment(
Type *Ty)
const ->
int {
2253 if (HST.isTypeForHVX(Ty))
2254 return HST.getVectorLength();
2255 return DL.getABITypeAlign(Ty).value();
2258auto HexagonVectorCombine::length(
Value *Val)
const ->
size_t {
2259 return length(Val->
getType());
2262auto HexagonVectorCombine::length(
Type *Ty)
const ->
size_t {
2264 assert(VecTy &&
"Must be a vector type");
2265 return VecTy->getElementCount().getFixedValue();
2268auto HexagonVectorCombine::getNullValue(
Type *Ty)
const ->
Constant * {
2276auto HexagonVectorCombine::getFullValue(
Type *Ty)
const ->
Constant * {
2284auto HexagonVectorCombine::getConstSplat(
Type *Ty,
int Val)
const
2288 Type *ElemTy = VecTy->getElementType();
2291 ConstantInt::get(ElemTy, Val));
2295auto HexagonVectorCombine::simplify(
Value *V)
const ->
Value * {
2297 SimplifyQuery Q(
DL, &TLI, &DT, &AC, In);
2304auto HexagonVectorCombine::insertb(IRBuilderBase &Builder,
Value *Dst,
2306 int Where)
const ->
Value * {
2307 assert(isByteVecTy(Dst->getType()) && isByteVecTy(Src->getType()));
2308 int SrcLen = getSizeOf(Src);
2309 int DstLen = getSizeOf(Dst);
2315 Value *P2Src = vresize(Builder, Src, P2Len,
Poison);
2316 Value *P2Dst = vresize(Builder, Dst, P2Len,
Poison);
2319 for (
int i = 0; i != P2Len; ++i) {
2323 (Where <= i && i < Where +
Length) ? P2Len + Start + (i - Where) : i;
2327 return vresize(Builder, P2Insert, DstLen,
Poison);
2330auto HexagonVectorCombine::vlalignb(IRBuilderBase &Builder,
Value *
Lo,
2332 assert(
Lo->getType() ==
Hi->getType() &&
"Argument type mismatch");
2335 int VecLen = getSizeOf(
Hi);
2336 if (
auto IntAmt = getIntValue(Amt))
2337 return getElementRange(Builder,
Lo,
Hi, VecLen - IntAmt->getSExtValue(),
2340 if (HST.isTypeForHVX(
Hi->getType())) {
2341 assert(
static_cast<unsigned>(VecLen) == HST.getVectorLength() &&
2342 "Expecting an exact HVX type");
2343 return createHvxIntrinsic(Builder, HST.getIntrinsicId(Hexagon::V6_vlalignb),
2344 Hi->getType(), {Hi, Lo, Amt});
2352 Builder.
CreateTrunc(Shift, Type::getInt32Ty(
F.getContext()),
"trn");
2357 return vralignb(Builder,
Lo,
Hi,
Sub);
2362auto HexagonVectorCombine::vralignb(IRBuilderBase &Builder,
Value *
Lo,
2364 assert(
Lo->getType() ==
Hi->getType() &&
"Argument type mismatch");
2367 int VecLen = getSizeOf(
Lo);
2368 if (
auto IntAmt = getIntValue(Amt))
2369 return getElementRange(Builder,
Lo,
Hi, IntAmt->getSExtValue(), VecLen);
2371 if (HST.isTypeForHVX(
Lo->getType())) {
2372 assert(
static_cast<unsigned>(VecLen) == HST.getVectorLength() &&
2373 "Expecting an exact HVX type");
2374 return createHvxIntrinsic(Builder, HST.getIntrinsicId(Hexagon::V6_valignb),
2375 Lo->getType(), {Hi, Lo, Amt});
2382 Builder.
CreateTrunc(Shift, Type::getInt32Ty(
F.getContext()),
"trn");
2386 Type *Int64Ty = Type::getInt64Ty(
F.getContext());
2398auto HexagonVectorCombine::concat(IRBuilderBase &Builder,
2402 std::vector<Value *> Work[2];
2403 int ThisW = 0, OtherW = 1;
2405 Work[ThisW].assign(Vecs.begin(), Vecs.end());
2406 while (Work[ThisW].
size() > 1) {
2408 SMask.
resize(length(Ty) * 2);
2409 std::iota(SMask.
begin(), SMask.
end(), 0);
2411 Work[OtherW].clear();
2412 if (Work[ThisW].
size() % 2 != 0)
2414 for (
int i = 0, e = Work[ThisW].
size(); i <
e; i += 2) {
2416 Work[ThisW][i], Work[ThisW][i + 1], SMask,
"shf");
2417 Work[OtherW].push_back(Joined);
2425 SMask.
resize(Vecs.size() * length(Vecs.front()->getType()));
2426 std::iota(SMask.
begin(), SMask.
end(), 0);
2431auto HexagonVectorCombine::vresize(IRBuilderBase &Builder,
Value *Val,
2435 assert(ValTy->getElementType() == Pad->getType());
2437 int CurSize = length(ValTy);
2438 if (CurSize == NewSize)
2441 if (CurSize > NewSize)
2442 return getElementRange(Builder, Val, Val, 0, NewSize);
2444 SmallVector<int, 128> SMask(NewSize);
2445 std::iota(SMask.
begin(), SMask.
begin() + CurSize, 0);
2446 std::fill(SMask.
begin() + CurSize, SMask.
end(), CurSize);
2451auto HexagonVectorCombine::rescale(IRBuilderBase &Builder,
Value *Mask,
2458 Type *FromSTy = FromTy->getScalarType();
2459 Type *ToSTy = ToTy->getScalarType();
2460 if (FromSTy == ToSTy)
2463 int FromSize = getSizeOf(FromSTy);
2464 int ToSize = getSizeOf(ToSTy);
2465 assert(FromSize % ToSize == 0 || ToSize % FromSize == 0);
2468 int FromCount = length(MaskTy);
2469 int ToCount = (FromCount * FromSize) / ToSize;
2470 assert((FromCount * FromSize) % ToSize == 0);
2472 auto *FromITy =
getIntTy(FromSize * 8);
2473 auto *ToITy =
getIntTy(ToSize * 8);
2478 Mask, VectorType::get(FromITy, FromCount,
false),
"sxt");
2480 Ext, VectorType::get(ToITy, ToCount,
false),
"cst");
2482 Cast, VectorType::get(getBoolTy(), ToCount,
false),
"trn");
2486auto HexagonVectorCombine::vlsb(IRBuilderBase &Builder,
Value *Val)
const
2489 if (ScalarTy == getBoolTy())
2492 Value *Bytes = vbytes(Builder, Val);
2494 return Builder.
CreateTrunc(Bytes, getBoolTy(getSizeOf(VecTy)),
"trn");
2497 return Builder.
CreateTrunc(Bytes, getBoolTy(),
"trn");
2501auto HexagonVectorCombine::vbytes(IRBuilderBase &Builder,
Value *Val)
const
2504 if (ScalarTy == getByteTy())
2507 if (ScalarTy != getBoolTy())
2508 return Builder.
CreateBitCast(Val, getByteTy(getSizeOf(Val)),
"cst");
2511 return Builder.
CreateSExt(Val, VectorType::get(getByteTy(), VecTy),
"sxt");
2512 return Builder.
CreateSExt(Val, getByteTy(),
"sxt");
2515auto HexagonVectorCombine::subvector(IRBuilderBase &Builder,
Value *Val,
2516 unsigned Start,
unsigned Length)
const
2519 return getElementRange(Builder, Val, Val, Start,
Length);
2522auto HexagonVectorCombine::sublo(IRBuilderBase &Builder,
Value *Val)
const
2524 size_t Len = length(Val);
2525 assert(Len % 2 == 0 &&
"Length should be even");
2526 return subvector(Builder, Val, 0, Len / 2);
2529auto HexagonVectorCombine::subhi(IRBuilderBase &Builder,
Value *Val)
const
2531 size_t Len = length(Val);
2532 assert(Len % 2 == 0 &&
"Length should be even");
2533 return subvector(Builder, Val, Len / 2, Len / 2);
2536auto HexagonVectorCombine::vdeal(IRBuilderBase &Builder,
Value *Val0,
2538 assert(Val0->getType() == Val1->getType());
2539 int Len = length(Val0);
2540 SmallVector<int, 128>
Mask(2 * Len);
2542 for (
int i = 0; i !=
Len; ++i) {
2549auto HexagonVectorCombine::vshuff(IRBuilderBase &Builder,
Value *Val0,
2551 assert(Val0->getType() == Val1->getType());
2552 int Len = length(Val0);
2553 SmallVector<int, 128>
Mask(2 * Len);
2555 for (
int i = 0; i !=
Len; ++i) {
2556 Mask[2 * i + 0] = i;
2562auto HexagonVectorCombine::createHvxIntrinsic(IRBuilderBase &Builder,
2568 auto getCast = [&](IRBuilderBase &Builder,
Value *Val,
2570 Type *SrcTy = Val->getType();
2571 if (SrcTy == DestTy)
2576 assert(HST.isTypeForHVX(SrcTy,
true));
2578 Type *BoolTy = Type::getInt1Ty(
F.getContext());
2583 unsigned HwLen = HST.getVectorLength();
2584 Intrinsic::ID TC = HwLen == 64 ? Intrinsic::hexagon_V6_pred_typecast
2585 : Intrinsic::hexagon_V6_pred_typecast_128B;
2595 for (
int i = 0, e =
Args.size(); i != e; ++i) {
2597 Type *
T = IntrTy->getParamType(i);
2598 if (
A->getType() !=
T) {
2604 StringRef MaybeName = !IntrTy->getReturnType()->isVoidTy() ?
"cup" :
"";
2605 CallInst *
Call = Builder.
CreateCall(IntrFn, IntrArgs, MaybeName);
2612 if (RetTy ==
nullptr || CallTy == RetTy)
2615 assert(HST.isTypeForHVX(CallTy,
true));
2616 return getCast(Builder,
Call, RetTy);
2619auto HexagonVectorCombine::splitVectorElements(IRBuilderBase &Builder,
2621 unsigned ToWidth)
const
2636 assert(VecTy->getElementType()->isIntegerTy());
2637 unsigned FromWidth = VecTy->getScalarSizeInBits();
2639 assert(ToWidth <= FromWidth &&
"Breaking up into wider elements?");
2640 unsigned NumResults = FromWidth / ToWidth;
2644 unsigned Length = length(VecTy);
2648 auto splitInHalf = [&](
unsigned Begin,
unsigned End,
auto splitFunc) ->
void {
2652 if (Begin + 1 == End)
2658 auto *VTy = VectorType::get(
getIntTy(Width / 2), 2 *
Length,
false);
2661 Value *Res =
vdeal(Builder, sublo(Builder, VVal), subhi(Builder, VVal));
2663 unsigned Half = (Begin + End) / 2;
2664 Results[Begin] = sublo(Builder, Res);
2665 Results[Half] = subhi(Builder, Res);
2667 splitFunc(Begin, Half, splitFunc);
2668 splitFunc(Half, End, splitFunc);
2671 splitInHalf(0, NumResults, splitInHalf);
2675auto HexagonVectorCombine::joinVectorElements(IRBuilderBase &Builder,
2677 VectorType *ToType)
const
2679 assert(ToType->getElementType()->isIntegerTy());
2690 unsigned ToWidth = ToType->getScalarSizeInBits();
2691 unsigned Width = Inputs.front()->getType()->getScalarSizeInBits();
2692 assert(Width <= ToWidth);
2694 unsigned Length = length(Inputs.front()->getType());
2696 unsigned NeedInputs = ToWidth / Width;
2697 if (Inputs.size() != NeedInputs) {
2702 Last, getConstSplat(
Last->getType(), Width - 1),
"asr");
2703 Inputs.resize(NeedInputs, Sign);
2706 while (Inputs.size() > 1) {
2709 for (
int i = 0, e = Inputs.size(); i < e; i += 2) {
2710 Value *Res =
vshuff(Builder, Inputs[i], Inputs[i + 1]);
2713 Inputs.resize(Inputs.size() / 2);
2716 assert(Inputs.front()->getType() == ToType);
2717 return Inputs.front();
2720auto HexagonVectorCombine::calculatePointerDifference(
Value *Ptr0,
2722 -> std::optional<int> {
2724 const SCEV *Scev0 = SE.getSCEV(Ptr0);
2725 const SCEV *Scev1 = SE.getSCEV(Ptr1);
2726 const SCEV *ScevDiff = SE.getMinusSCEV(Scev0, Scev1);
2728 APInt
V =
Const->getAPInt();
2729 if (
V.isSignedIntN(8 *
sizeof(
int)))
2730 return static_cast<int>(
V.getSExtValue());
2737 I->eraseFromParent();
2739 SmallVector<Instruction *, 8> ToErase;
2742#define CallBuilder(B, F) \
2745 if (auto *I = dyn_cast<Instruction>(V)) \
2746 B_.ToErase.push_back(I); \
2750 auto Simplify = [
this](
Value *
V) {
2756 auto StripBitCast = [](
Value *
V) {
2758 V =
C->getOperand(0);
2762 Ptr0 = StripBitCast(Ptr0);
2763 Ptr1 = StripBitCast(Ptr1);
2765 return std::nullopt;
2769 if (Gep0->getPointerOperand() != Gep1->getPointerOperand())
2770 return std::nullopt;
2771 if (Gep0->getSourceElementType() != Gep1->getSourceElementType())
2772 return std::nullopt;
2774 Builder
B(Gep0->getParent());
2775 int Scale = getSizeOf(Gep0->getSourceElementType(),
Alloc);
2778 if (Gep0->getNumOperands() != 2 || Gep1->getNumOperands() != 2)
2779 return std::nullopt;
2781 Value *Idx0 = Gep0->getOperand(1);
2782 Value *Idx1 = Gep1->getOperand(1);
2787 return Diff->getSExtValue() * Scale;
2789 KnownBits Known0 = getKnownBits(Idx0, Gep0);
2790 KnownBits Known1 = getKnownBits(Idx1, Gep1);
2793 return std::nullopt;
2801 Diff0 =
C->getSExtValue();
2803 return std::nullopt;
2812 Diff1 =
C->getSExtValue();
2814 return std::nullopt;
2817 return (Diff0 + Diff1) * Scale;
2822auto HexagonVectorCombine::getNumSignificantBits(
const Value *V,
2823 const Instruction *CtxI)
const
2828auto HexagonVectorCombine::getKnownBits(
const Value *V,
2829 const Instruction *CtxI)
const
2834auto HexagonVectorCombine::isSafeToClone(
const Instruction &In)
const ->
bool {
2835 if (
In.mayHaveSideEffects() ||
In.isAtomic() ||
In.isVolatile() ||
2836 In.isFenceLike() ||
In.mayReadOrWriteMemory()) {
2844template <
typename T>
2845auto HexagonVectorCombine::isSafeToMoveBeforeInBB(
const Instruction &In,
2847 const T &IgnoreInsts)
const
2850 [
this](
const Instruction &
I) -> std::optional<MemoryLocation> {
2852 switch (
II->getIntrinsicID()) {
2853 case Intrinsic::masked_load:
2855 case Intrinsic::masked_store:
2871 bool MayWrite =
In.mayWriteToMemory();
2872 auto MaybeLoc = getLocOrNone(In);
2874 auto From =
In.getIterator();
2877 bool MoveUp = (To !=
Block.end() && To->comesBefore(&In));
2879 MoveUp ? std::make_pair(To, From) : std::make_pair(std::next(From), To);
2880 for (
auto It =
Range.first; It !=
Range.second; ++It) {
2881 const Instruction &I = *It;
2882 if (llvm::is_contained(IgnoreInsts, &I))
2885 if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
2886 if (II->getIntrinsicID() == Intrinsic::assume)
2893 if (!CB->hasFnAttr(Attribute::WillReturn))
2895 if (!CB->hasFnAttr(Attribute::NoSync))
2898 if (
I.mayReadOrWriteMemory()) {
2899 auto MaybeLocI = getLocOrNone(I);
2900 if (MayWrite || I.mayWriteToMemory()) {
2901 if (!MaybeLoc || !MaybeLocI)
2903 if (!AA.isNoAlias(*MaybeLoc, *MaybeLocI))
2911auto HexagonVectorCombine::isByteVecTy(
Type *Ty)
const ->
bool {
2913 return VecTy->getElementType() == getByteTy();
2917auto HexagonVectorCombine::getElementRange(IRBuilderBase &Builder,
Value *
Lo,
2921 SmallVector<int, 128> SMask(
Length);
2922 std::iota(SMask.
begin(), SMask.
end(), Start);
2929class HexagonVectorCombineLegacy :
public FunctionPass {
2933 HexagonVectorCombineLegacy() : FunctionPass(
ID) {}
2935 StringRef getPassName()
const override {
return "Hexagon Vector Combine"; }
2937 void getAnalysisUsage(AnalysisUsage &AU)
const override {
2945 FunctionPass::getAnalysisUsage(AU);
2949 if (skipFunction(
F))
2951 AliasAnalysis &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
2952 AssumptionCache &AC =
2953 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
F);
2954 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
2955 ScalarEvolution &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
2956 TargetLibraryInfo &TLI =
2957 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
F);
2958 auto &
TM = getAnalysis<TargetPassConfig>().getTM<HexagonTargetMachine>();
2959 HexagonVectorCombine HVC(
F, AA, AC, DT, SE, TLI, TM);
2965char HexagonVectorCombineLegacy::ID = 0;
2968 "Hexagon Vector Combine",
false,
false)
2979 return new HexagonVectorCombineLegacy();
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Prepare AGPR Alloc
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
static IntegerType * getIntTy(IRBuilderBase &B, const TargetLibraryInfo *TLI)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
This file defines the DenseMap class.
static bool runOnFunction(Function &F, bool PostInlining)
static cl::opt< unsigned > SizeLimit("eif-limit", cl::init(6), cl::Hidden, cl::desc("Size limit in Hexagon early if-conversion"))
#define CallBuilder(B, F)
static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
static bool isCandidate(const MachineInstr *MI, Register &DefedReg, Register FrameReg)
static bool isUndef(const MachineInstr &MI)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Remove Loads Into Fake Uses
static ConstantInt * getConstInt(MDNode *MD, unsigned NumOp)
This file defines the SmallVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static SymbolRef::Type getType(const Symbol *Sym)
Target-Independent Code Generator Pass Configuration Options pass.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
AnalysisUsage & addRequired()
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
InstListType::const_iterator const_iterator
InstListType::iterator iterator
Instruction iterators...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
AttributeList getAttributes() const
Return the attributes for this call.
@ ICMP_ULT
unsigned less than
This is the shared class of boolean and integer constants.
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static LLVM_ABI Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
iterator_range< iterator > children()
DomTreeNodeBase< NodeT > * getRootNode()
getRootNode - This returns the entry node for the CFG of the function.
Legacy analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
FunctionPass class - This class is used to implement most global optimizations.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
const BasicBlock & back() const
bool isHVXVectorType(EVT VecTy, bool IncludeBool=false) const
bool useHVXV62Ops() const
bool useHVXV69Ops() const
unsigned getVectorLength() const
bool useHVXV66Ops() const
bool isTypeForHVX(Type *VecTy, bool IncludeBool=false) const
Intrinsic::ID getIntrinsicId(unsigned Opc) const
Common base class shared among various IRBuilders.
LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
LLVM_ABI Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)
const char * getOpcodeName() const
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
An instruction for reading from memory.
bool doesNotAccessMemory() const
Whether this function accesses no memory.
bool onlyAccessesInaccessibleMem() const
Whether this function only (at most) accesses inaccessible memory.
static LLVM_ABI std::optional< MemoryLocation > getOrNone(const Instruction *Inst)
static LLVM_ABI MemoryLocation getForArgument(const CallBase *Call, unsigned ArgIdx, const TargetLibraryInfo *TLI)
Return a location representing a particular argument of a call.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
The main scalar evolution driver.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Provides information about what library functions are available for the current target.
Primary interface to the complete machine description for the target machine.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
Target-Independent Code Generator Pass Configuration Options.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
const ParentTy * getParent() const
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Abstract Attribute helper functions.
Rounding
Possible values of current rounding mode, which is specified in bits 23:22 of FPCR.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ BasicBlock
Various leaf nodes.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
bool match(Val *V, const Pattern &P)
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
@ Undef
Value of the register doesn't matter.
initializer< Ty > init(const Ty &Val)
@ User
could "use" a pointer
friend class Instruction
Iterator for Instructions in a `BasicBlock.
LLVM_ABI Instruction * getTerminator() const
LLVM_ABI Instruction & front() const
This is an optimization pass for GlobalISel generic memory operations.
FunctionPass * createHexagonVectorCombineLegacyPass()
FunctionAddr VTableAddr Value
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
LLVM_ABI bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
If the specified value is a trivially dead instruction, delete it.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
MemoryEffectsBase< IRMemLocation > MemoryEffects
Summary of how a function affects memory in the program.
LLVM_ABI Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,...
OutputIt copy_if(R &&Range, OutputIt Out, UnaryPredicate P)
Provide wrappers to std::copy_if which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
detail::concat_range< ValueT, RangeTs... > concat(RangeTs &&...Ranges)
Returns a concatenated range across two or more ranges.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
LLVM_ABI Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
DomTreeNodeBase< BasicBlock > DomTreeNode
auto reverse(ContainerTy &&C)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
DWARFExpression::Operation Op
auto max_element(R &&Range)
Provide wrappers to std::max_element which take ranges instead of having to pass begin/end explicitly...
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
LLVM_ABI unsigned ComputeMaxSignificantBits(const Value *Op, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Get the upper bound on bit size for this Value Op as a signed integer.
AAResults AliasAnalysis
Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.
LLVM_ABI bool mayHaveNonDefUseDependency(const Instruction &I)
Returns true if the result or effects of the given instructions I depend values not reachable through...
MaskT vshuff(ArrayRef< int > Vu, ArrayRef< int > Vv, unsigned Size, bool TakeOdd)
MaskT vdeal(ArrayRef< int > Vu, ArrayRef< int > Vv, unsigned Size, bool TakeOdd)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.