34#include "llvm/IR/IntrinsicsHexagon.h"
58#define DEBUG_TYPE "hexagon-vc"
73class HexagonVectorCombine {
78 :
F(F_),
DL(
F.getDataLayout()),
AA(AA_), AC(AC_), DT(DT_),
88 Type *getByteTy(
int ElemCount = 0)
const;
91 Type *getBoolTy(
int ElemCount = 0)
const;
95 std::optional<APInt> getIntValue(
const Value *Val)
const;
101 bool isTrue(
const Value *Val)
const;
103 bool isFalse(
const Value *Val)
const;
112 int getSizeOf(
const Value *Val, SizeKind Kind = Store)
const;
113 int getSizeOf(
const Type *Ty, SizeKind Kind = Store)
const;
114 int getTypeAlignment(
Type *Ty)
const;
115 size_t length(
Value *Val)
const;
116 size_t length(
Type *Ty)
const;
125 int Length,
int Where)
const;
149 unsigned ToWidth)
const;
153 std::optional<int> calculatePointerDifference(
Value *Ptr0,
Value *Ptr1)
const;
155 unsigned getNumSignificantBits(
const Value *V,
162 template <
typename T = std::vector<Instruction *>>
165 const T &IgnoreInsts = {})
const;
168 [[maybe_unused]]
bool isByteVecTy(
Type *Ty)
const;
181 int Start,
int Length)
const;
200 AlignVectors(
const HexagonVectorCombine &HVC_) : HVC(HVC_) {}
205 using InstList = std::vector<Instruction *>;
209 AddrInfo(
const AddrInfo &) =
default;
212 : Inst(
I), Addr(
A), ValTy(
T), HaveAlign(
H),
213 NeedAlign(HVC.getTypeAlignment(ValTy)) {}
214 AddrInfo &operator=(
const AddrInfo &) =
default;
225 using AddrList = std::vector<AddrInfo>;
229 return A->comesBefore(
B);
232 using DepList = std::set<Instruction *, InstrLess>;
235 MoveGroup(
const AddrInfo &AI,
Instruction *
B,
bool Hvx,
bool Load)
236 :
Base(
B), Main{AI.Inst}, Clones{}, IsHvx(Hvx), IsLoad(Load) {}
237 MoveGroup() =
default;
245 using MoveList = std::vector<MoveGroup>;
265 Segment(
Value *Val,
int Begin,
int Len)
266 : Val(Val), Start(Begin),
Size(Len) {}
267 Segment(
const Segment &Seg) =
default;
268 Segment &operator=(
const Segment &Seg) =
default;
275 Block(
Value *Val,
int Len,
int Pos) : Seg(Val, 0, Len), Pos(Pos) {}
277 : Seg(Val, Off, Len), Pos(Pos) {}
285 ByteSpan section(
int Start,
int Length)
const;
286 ByteSpan &shift(
int Offset);
289 int size()
const {
return Blocks.size(); }
290 Block &operator[](
int i) {
return Blocks[i]; }
291 const Block &operator[](
int i)
const {
return Blocks[i]; }
293 std::vector<Block> Blocks;
296 iterator begin() {
return Blocks.begin(); }
297 iterator end() {
return Blocks.end(); }
303 Align getAlignFromValue(
const Value *V)
const;
304 std::optional<AddrInfo> getAddrInfo(
Instruction &In)
const;
305 bool isHvx(
const AddrInfo &AI)
const;
307 [[maybe_unused]]
bool isSectorTy(
Type *Ty)
const;
315 const InstMap &CloneMap = InstMap())
const;
318 const InstMap &CloneMap = InstMap())
const;
342 bool createAddressGroups();
343 MoveList createLoadGroups(
const AddrList &Group)
const;
344 MoveList createStoreGroups(
const AddrList &Group)
const;
345 bool moveTogether(MoveGroup &Move)
const;
346 template <
typename T>
349 void realignLoadGroup(
IRBuilderBase &Builder,
const ByteSpan &VSpan,
350 int ScLen,
Value *AlignVal,
Value *AlignAddr)
const;
351 void realignStoreGroup(
IRBuilderBase &Builder,
const ByteSpan &VSpan,
352 int ScLen,
Value *AlignVal,
Value *AlignAddr)
const;
353 bool realignGroup(
const MoveGroup &Move)
const;
356 int Alignment)
const;
363 std::map<Instruction *, AddrList> AddrGroups;
364 const HexagonVectorCombine &HVC;
369 OS <<
"Inst: " << AI.Inst <<
" " << *AI.Inst <<
'\n';
370 OS <<
"Addr: " << *AI.Addr <<
'\n';
371 OS <<
"Type: " << *AI.ValTy <<
'\n';
372 OS <<
"HaveAlign: " << AI.HaveAlign.
value() <<
'\n';
373 OS <<
"NeedAlign: " << AI.NeedAlign.
value() <<
'\n';
374 OS <<
"Offset: " << AI.Offset;
380 OS <<
"IsLoad:" << (MG.IsLoad ?
"yes" :
"no");
381 OS <<
", IsHvx:" << (MG.IsHvx ?
"yes" :
"no") <<
'\n';
384 OS <<
" " << *
I <<
'\n';
387 OS <<
" " << *
I <<
'\n';
389 for (
auto [K, V] : MG.Clones) {
391 K->printAsOperand(OS,
false);
392 OS <<
"\t-> " << *V <<
'\n';
399 const AlignVectors::ByteSpan::Block &
B) {
400 OS <<
" @" <<
B.Pos <<
" [" <<
B.Seg.Start <<
',' <<
B.Seg.Size <<
"] ";
401 if (
B.Seg.Val ==
reinterpret_cast<const Value *
>(&
B)) {
402 OS <<
"(self:" <<
B.Seg.Val <<
')';
403 }
else if (
B.Seg.Val !=
nullptr) {
413 OS <<
"ByteSpan[size=" << BS.size() <<
", extent=" << BS.extent() <<
'\n';
414 for (
const AlignVectors::ByteSpan::Block &
B : BS)
422 HvxIdioms(
const HexagonVectorCombine &HVC_) : HVC(HVC_) {
423 auto *
Int32Ty = HVC.getIntTy(32);
424 HvxI32Ty = HVC.getHvxTy(
Int32Ty,
false);
425 HvxP32Ty = HVC.getHvxTy(
Int32Ty,
true);
446 std::optional<unsigned> RoundAt;
451 -> std::pair<unsigned, Signedness>;
452 auto canonSgn(SValue
X, SValue
Y)
const -> std::pair<SValue, SValue>;
454 auto matchFxpMul(
Instruction &In)
const -> std::optional<FxpOp>;
458 const FxpOp &
Op)
const ->
Value *;
460 bool Rounding)
const ->
Value *;
462 bool Rounding)
const ->
Value *;
465 Value *CarryIn =
nullptr)
const
466 -> std::pair<Value *, Value *>;
471 -> std::pair<Value *, Value *>;
480 const HexagonVectorCombine &HVC;
486 const HvxIdioms::FxpOp &
Op) {
487 static const char *SgnNames[] = {
"Positive",
"Signed",
"Unsigned"};
489 if (
Op.RoundAt.has_value()) {
490 if (
Op.Frac != 0 && *
Op.RoundAt ==
Op.Frac - 1) {
493 OS <<
" + 1<<" << *
Op.RoundAt;
496 OS <<
"\n X:(" << SgnNames[
Op.X.Sgn] <<
") " << *
Op.X.Val <<
"\n"
497 <<
" Y:(" << SgnNames[
Op.Y.Sgn] <<
") " << *
Op.Y.Val;
505template <
typename T>
T *getIfUnordered(
T *MaybeT) {
506 return MaybeT && MaybeT->isUnordered() ? MaybeT :
nullptr;
518#if !defined(_MSC_VER) || _MSC_VER >= 1926
522template <
typename Pred,
typename... Ts>
523void erase_if(std::map<Ts...> &map, Pred p)
525template <
typename Pred,
typename T,
typename U>
526void erase_if(std::map<T, U> &map, Pred p)
529 for (
auto i = map.begin(), e = map.end(); i != e;) {
538template <
typename Pred,
typename T>
void erase_if(
T &&container, Pred p) {
576auto AlignVectors::ByteSpan::extent()
const ->
int {
579 int Min = Blocks[0].Pos;
580 int Max = Blocks[0].Pos + Blocks[0].Seg.Size;
581 for (
int i = 1, e =
size(); i !=
e; ++i) {
582 Min = std::min(Min, Blocks[i].Pos);
583 Max = std::max(Max, Blocks[i].Pos + Blocks[i].Seg.Size);
588auto AlignVectors::ByteSpan::section(
int Start,
int Length)
const -> ByteSpan {
590 for (
const ByteSpan::Block &
B : Blocks) {
591 int L = std::max(
B.Pos, Start);
592 int R = std::min(
B.Pos +
B.Seg.Size, Start +
Length);
595 int Off =
L >
B.Pos ?
L -
B.Pos : 0;
596 Section.Blocks.emplace_back(
B.Seg.Val,
B.Seg.Start + Off, R - L, L);
602auto AlignVectors::ByteSpan::shift(
int Offset) -> ByteSpan & {
609 SmallVector<Value *, 8> Values(Blocks.size());
610 for (
int i = 0, e = Blocks.size(); i != e; ++i)
611 Values[i] = Blocks[i].Seg.Val;
615auto AlignVectors::getAlignFromValue(
const Value *V)
const ->
Align {
617 assert(
C &&
"Alignment must be a compile-time constant integer");
618 return C->getAlignValue();
621auto AlignVectors::getAddrInfo(Instruction &In)
const
622 -> std::optional<AddrInfo> {
624 return AddrInfo(HVC, L,
L->getPointerOperand(),
L->getType(),
627 return AddrInfo(HVC, S, S->getPointerOperand(),
628 S->getValueOperand()->getType(), S->getAlign());
632 case Intrinsic::masked_load:
633 return AddrInfo(HVC,
II,
II->getArgOperand(0),
II->getType(),
634 getAlignFromValue(
II->getArgOperand(1)));
635 case Intrinsic::masked_store:
636 return AddrInfo(HVC,
II,
II->getArgOperand(1),
637 II->getArgOperand(0)->getType(),
638 getAlignFromValue(
II->getArgOperand(2)));
644auto AlignVectors::isHvx(
const AddrInfo &AI)
const ->
bool {
648auto AlignVectors::getPayload(
Value *Val)
const ->
Value * {
652 ID =
II->getIntrinsicID();
654 return In->getOperand(0);
659auto AlignVectors::getMask(
Value *Val)
const ->
Value * {
661 switch (
II->getIntrinsicID()) {
662 case Intrinsic::masked_load:
663 return II->getArgOperand(2);
664 case Intrinsic::masked_store:
665 return II->getArgOperand(3);
669 Type *ValTy = getPayload(Val)->getType();
671 return HVC.getFullValue(HVC.getBoolTy(HVC.length(VecTy)));
672 return HVC.getFullValue(HVC.getBoolTy());
675auto AlignVectors::getPassThrough(
Value *Val)
const ->
Value * {
677 if (
II->getIntrinsicID() == Intrinsic::masked_load)
678 return II->getArgOperand(3);
683auto AlignVectors::createAdjustedPointer(IRBuilderBase &Builder,
Value *
Ptr,
684 Type *ValTy,
int Adjust,
685 const InstMap &CloneMap)
const
688 if (Instruction *New = CloneMap.lookup(
I))
690 return Builder.CreatePtrAdd(
Ptr, HVC.getConstInt(Adjust),
"gep");
693auto AlignVectors::createAlignedPointer(IRBuilderBase &Builder,
Value *
Ptr,
694 Type *ValTy,
int Alignment,
695 const InstMap &CloneMap)
const
699 for (
auto [Old, New] : CloneMap)
700 I->replaceUsesOfWith(Old, New);
705 Value *AsInt = Builder.CreatePtrToInt(
Ptr, HVC.getIntTy(),
"pti");
706 Value *
Mask = HVC.getConstInt(-Alignment);
707 Value *
And = Builder.CreateAnd(remap(AsInt), Mask,
"and");
708 return Builder.CreateIntToPtr(
712auto AlignVectors::createLoad(IRBuilderBase &Builder,
Type *ValTy,
Value *
Ptr,
720 "Expectning scalar predicate");
721 if (HVC.isFalse(Predicate))
723 if (!HVC.isTrue(Predicate) && HvxHasPredLoad) {
724 Value *
Load = createPredicatedLoad(Builder, ValTy,
Ptr, Predicate,
725 Alignment, MDSources);
726 return Builder.CreateSelect(Mask, Load, PassThru);
730 assert(!HVC.isUndef(Mask));
731 if (HVC.isZero(Mask))
733 if (HVC.isTrue(Mask))
734 return createSimpleLoad(Builder, ValTy,
Ptr, Alignment, MDSources);
737 Mask, PassThru,
"mld");
742auto AlignVectors::createSimpleLoad(IRBuilderBase &Builder,
Type *ValTy,
747 Builder.CreateAlignedLoad(ValTy,
Ptr,
Align(Alignment),
"ald");
752auto AlignVectors::createPredicatedLoad(IRBuilderBase &Builder,
Type *ValTy,
758 "Predicates 'scalar' vector loads not yet supported");
760 assert(!
Predicate->getType()->isVectorTy() &&
"Expectning scalar predicate");
761 assert(HVC.getSizeOf(ValTy, HVC.Alloc) % Alignment == 0);
762 if (HVC.isFalse(Predicate))
764 if (HVC.isTrue(Predicate))
765 return createSimpleLoad(Builder, ValTy,
Ptr, Alignment, MDSources);
767 auto V6_vL32b_pred_ai = HVC.HST.
getIntrinsicId(Hexagon::V6_vL32b_pred_ai);
769 return HVC.createHvxIntrinsic(Builder, V6_vL32b_pred_ai, ValTy,
774auto AlignVectors::createStore(IRBuilderBase &Builder,
Value *Val,
Value *
Ptr,
777 if (HVC.isZero(Mask) || HVC.isUndef(Val) || HVC.isUndef(Mask))
780 "Expectning scalar predicate"));
782 if (HVC.isFalse(Predicate))
784 if (HVC.isTrue(Predicate))
789 if (HVC.isTrue(Mask)) {
791 return createPredicatedStore(Builder, Val,
Ptr, Predicate, Alignment,
795 return createSimpleStore(Builder, Val,
Ptr, Alignment, MDSources);
801 Builder.CreateMaskedStore(Val,
Ptr,
Align(Alignment), Mask);
808 Value *PredLoad = createPredicatedLoad(Builder, Val->getType(),
Ptr,
809 Predicate, Alignment, MDSources);
810 Value *Mux = Builder.CreateSelect(Mask, Val, PredLoad);
811 return createPredicatedStore(Builder, Mux,
Ptr, Predicate, Alignment,
815auto AlignVectors::createSimpleStore(IRBuilderBase &Builder,
Value *Val,
824auto AlignVectors::createPredicatedStore(IRBuilderBase &Builder,
Value *Val,
830 "Predicates 'scalar' vector stores not yet supported");
832 if (HVC.isFalse(Predicate))
834 if (HVC.isTrue(Predicate))
835 return createSimpleStore(Builder, Val,
Ptr, Alignment, MDSources);
837 assert(HVC.getSizeOf(Val, HVC.Alloc) % Alignment == 0);
838 auto V6_vS32b_pred_ai = HVC.HST.
getIntrinsicId(Hexagon::V6_vS32b_pred_ai);
840 return HVC.createHvxIntrinsic(Builder, V6_vS32b_pred_ai,
nullptr,
845auto AlignVectors::getUpwardDeps(Instruction *In, Instruction *
Base)
const
849 "Base and In should be in the same block");
850 assert(
Base->comesBefore(In) &&
"Base should come before In");
853 std::deque<Instruction *> WorkQ = {
In};
854 while (!WorkQ.empty()) {
861 if (
I->getParent() == Parent &&
Base->comesBefore(
I))
869auto AlignVectors::createAddressGroups() ->
bool {
874 auto findBaseAndOffset = [&](AddrInfo &AI) -> std::pair<Instruction *, int> {
875 for (AddrInfo &W : WorkStack) {
876 if (
auto D = HVC.calculatePointerDifference(AI.Addr,
W.Addr))
877 return std::make_pair(
W.Inst, *
D);
879 return std::make_pair(
nullptr, 0);
882 auto traverseBlock = [&](
DomTreeNode *DomN,
auto Visit) ->
void {
884 for (Instruction &
I :
Block) {
885 auto AI = this->getAddrInfo(
I);
888 auto F = findBaseAndOffset(*AI);
890 if (Instruction *BI =
F.first) {
891 AI->Offset =
F.second;
894 WorkStack.push_back(*AI);
895 GroupInst = AI->Inst;
897 AddrGroups[GroupInst].push_back(*AI);
903 while (!WorkStack.empty() && WorkStack.back().Inst->getParent() == &
Block)
904 WorkStack.pop_back();
907 traverseBlock(HVC.DT.
getRootNode(), traverseBlock);
908 assert(WorkStack.empty());
913 erase_if(AddrGroups, [](
auto &
G) {
return G.second.size() == 1; });
917 G.second, [&](
auto &
I) { return HVC.HST.isTypeForHVX(I.ValTy); });
920 return !AddrGroups.empty();
923auto AlignVectors::createLoadGroups(
const AddrList &Group)
const -> MoveList {
931 auto tryAddTo = [&](
const AddrInfo &
Info, MoveGroup &Move) {
932 assert(!Move.Main.empty() &&
"Move group should have non-empty Main");
936 if (Move.IsHvx != isHvx(
Info))
940 if (
Base->getParent() !=
Info.Inst->getParent())
943 if (!HVC.isSafeToMoveBeforeInBB(*
Info.Inst,
Base->getIterator()))
947 return HVC.isSafeToMoveBeforeInBB(*
I,
Base->getIterator()) &&
948 HVC.isSafeToClone(*
I);
950 DepList Deps = getUpwardDeps(
Info.Inst,
Base);
954 Move.Main.push_back(
Info.Inst);
961 for (
const AddrInfo &
Info : Group) {
962 if (!
Info.Inst->mayReadFromMemory())
964 if (LoadGroups.empty() || !tryAddTo(
Info, LoadGroups.back()))
965 LoadGroups.emplace_back(
Info, Group.front().Inst, isHvx(
Info),
true);
969 erase_if(LoadGroups, [](
const MoveGroup &
G) {
return G.Main.size() <= 1; });
973 erase_if(LoadGroups, [](
const MoveGroup &
G) {
return G.IsHvx; });
978auto AlignVectors::createStoreGroups(
const AddrList &Group)
const -> MoveList {
986 auto tryAddTo = [&](
const AddrInfo &
Info, MoveGroup &Move) {
987 assert(!Move.Main.empty() &&
"Move group should have non-empty Main");
993 "Not handling stores with return values");
995 if (Move.IsHvx != isHvx(
Info))
1001 if (
Base->getParent() !=
Info.Inst->getParent())
1003 if (!HVC.isSafeToMoveBeforeInBB(*
Info.Inst,
Base->getIterator(), Move.Main))
1005 Move.Main.push_back(
Info.Inst);
1009 MoveList StoreGroups;
1011 for (
auto I = Group.rbegin(),
E = Group.rend();
I !=
E; ++
I) {
1012 const AddrInfo &
Info = *
I;
1013 if (!
Info.Inst->mayWriteToMemory())
1015 if (StoreGroups.empty() || !tryAddTo(
Info, StoreGroups.back()))
1016 StoreGroups.emplace_back(
Info, Group.front().Inst, isHvx(
Info),
false);
1020 erase_if(StoreGroups, [](
const MoveGroup &
G) {
return G.Main.size() <= 1; });
1024 erase_if(StoreGroups, [](
const MoveGroup &
G) {
return G.IsHvx; });
1029 if (!VADoFullStores) {
1030 erase_if(StoreGroups, [
this](
const MoveGroup &
G) {
1032 auto MaybeInfo = this->getAddrInfo(*S);
1033 assert(MaybeInfo.has_value());
1034 return HVC.HST.isHVXVectorType(
1035 EVT::getEVT(MaybeInfo->ValTy, false));
1043auto AlignVectors::moveTogether(MoveGroup &Move)
const ->
bool {
1045 assert(!Move.Main.empty() &&
"Move group should have non-empty Main");
1051 Move.Clones = cloneBefore(Where->
getIterator(), Move.Deps);
1054 for (Instruction *M : Main) {
1056 M->moveAfter(Where);
1057 for (
auto [Old, New] : Move.Clones)
1058 M->replaceUsesOfWith(Old, New);
1062 for (
int i = 0, e = Move.Deps.size(); i != e; ++i)
1063 Move.Deps[i] = Move.Clones[Move.Deps[i]];
1068 assert(Move.Deps.empty());
1071 for (Instruction *M : Main.drop_front(1)) {
1077 return Move.Main.size() + Move.Deps.size() > 1;
1080template <
typename T>
1085 for (Instruction *
I : Insts) {
1086 assert(HVC.isSafeToClone(*
I));
1088 C->setName(Twine(
"c.") +
I->getName() +
".");
1089 C->insertBefore(To);
1091 for (
auto [Old, New] : Map)
1092 C->replaceUsesOfWith(Old, New);
1093 Map.insert(std::make_pair(
I,
C));
1098auto AlignVectors::realignLoadGroup(IRBuilderBase &Builder,
1099 const ByteSpan &VSpan,
int ScLen,
1104 Type *SecTy = HVC.getByteTy(ScLen);
1105 int NumSectors = (VSpan.extent() + ScLen - 1) / ScLen;
1106 bool DoAlign = !HVC.isZero(AlignVal);
1108 BasicBlock *BaseBlock = Builder.GetInsertBlock();
1111 auto *True = HVC.getFullValue(HVC.getBoolTy(ScLen));
1138 for (
int Index = 0;
Index != NumSectors; ++
Index)
1139 ASpan.Blocks.emplace_back(
nullptr, ScLen, Index * ScLen);
1140 for (
int Index = 0;
Index != NumSectors; ++
Index) {
1141 ASpan.Blocks[
Index].Seg.Val =
1142 reinterpret_cast<Value *
>(&ASpan.Blocks[
Index]);
1148 DenseMap<void *, Instruction *> EarliestUser;
1154 assert(
A->getParent() ==
B->getParent());
1155 return A->comesBefore(
B);
1157 auto earliestUser = [&](
const auto &
Uses) {
1159 for (
const Use &U :
Uses) {
1161 assert(
I !=
nullptr &&
"Load used in a non-instruction?");
1165 if (
I->getParent() == BaseBlock) {
1167 User = std::min(User,
I, isEarlier);
1175 for (
const ByteSpan::Block &
B : VSpan) {
1176 ByteSpan ASection = ASpan.section(
B.Pos,
B.Seg.Size);
1177 for (
const ByteSpan::Block &S : ASection) {
1178 auto &EU = EarliestUser[S.Seg.Val];
1179 EU = std::min(EU, earliestUser(
B.Seg.Val->uses()), isEarlier);
1184 dbgs() <<
"ASpan:\n" << ASpan <<
'\n';
1185 dbgs() <<
"Earliest users of ASpan:\n";
1186 for (
auto &[Val, User] : EarliestUser) {
1187 dbgs() << Val <<
"\n ->" << *
User <<
'\n';
1191 auto createLoad = [&](IRBuilderBase &Builder,
const ByteSpan &VSpan,
1192 int Index,
bool MakePred) {
1194 createAdjustedPointer(Builder, AlignAddr, SecTy, Index * ScLen);
1196 MakePred ? makeTestIfUnaligned(Builder, AlignVal, ScLen) : nullptr;
1201 int Width = (1 + DoAlign) * ScLen;
1202 return this->createLoad(Builder, SecTy,
Ptr, Predicate, ScLen, True, Undef,
1203 VSpan.section(Start, Width).values());
1208 assert(
In->getParent() == To->getParent());
1209 DepList Deps = getUpwardDeps(&*In, &*To);
1212 InstMap
Map = cloneBefore(In, Deps);
1213 for (
auto [Old, New] : Map)
1214 In->replaceUsesOfWith(Old, New);
1219 for (
int Index = 0;
Index != NumSectors + 1; ++
Index) {
1227 DoAlign &&
Index > 0 ? EarliestUser[&ASpan[
Index - 1]] :
nullptr;
1229 Index < NumSectors ? EarliestUser[&ASpan[
Index]] :
nullptr;
1230 if (
auto *Where = std::min(PrevAt, ThisAt, isEarlier)) {
1233 createLoad(Builder, VSpan, Index, DoAlign && Index == NumSectors);
1241 if (!HVC.isSafeToMoveBeforeInBB(*Load, BasePos))
1242 moveBefore(
Load->getIterator(), BasePos);
1244 LLVM_DEBUG(
dbgs() <<
"Loads[" << Index <<
"]:" << *Loads[Index] <<
'\n');
1250 for (
int Index = 0;
Index != NumSectors; ++
Index) {
1251 ASpan[
Index].Seg.Val =
nullptr;
1252 if (
auto *Where = EarliestUser[&ASpan[Index]]) {
1258 assert(NextLoad !=
nullptr);
1259 Val = HVC.vralignb(Builder, Val, NextLoad, AlignVal);
1261 ASpan[
Index].Seg.Val = Val;
1266 for (
const ByteSpan::Block &
B : VSpan) {
1267 ByteSpan ASection = ASpan.section(
B.Pos,
B.Seg.Size).shift(-
B.Pos);
1274 std::vector<ByteSpan::Block *> ABlocks;
1275 for (ByteSpan::Block &S : ASection) {
1276 if (S.Seg.Val !=
nullptr)
1277 ABlocks.push_back(&S);
1280 [&](
const ByteSpan::Block *
A,
const ByteSpan::Block *
B) {
1284 for (ByteSpan::Block *S : ABlocks) {
1289 Value *Pay = HVC.vbytes(Builder, getPayload(S->Seg.Val));
1291 HVC.insertb(Builder, Accum, Pay, S->Seg.Start, S->Seg.Size, S->Pos);
1299 Type *ValTy = getPayload(
B.Seg.Val)->getType();
1302 getPassThrough(
B.Seg.Val),
"sel");
1307auto AlignVectors::realignStoreGroup(IRBuilderBase &Builder,
1308 const ByteSpan &VSpan,
int ScLen,
1313 Type *SecTy = HVC.getByteTy(ScLen);
1314 int NumSectors = (VSpan.extent() + ScLen - 1) / ScLen;
1315 bool DoAlign = !HVC.isZero(AlignVal);
1318 ByteSpan ASpanV, ASpanM;
1322 auto MakeVec = [](IRBuilderBase &Builder,
Value *Val) ->
Value * {
1326 auto *VecTy = VectorType::get(Ty, 1,
false);
1332 for (
int Index = (DoAlign ? -1 : 0);
Index != NumSectors + DoAlign; ++
Index) {
1336 VSpan.section(Index * ScLen, ScLen).shift(-Index * ScLen);
1341 for (ByteSpan::Block &S : VSection) {
1342 Value *Pay = getPayload(S.Seg.Val);
1344 Pay->
getType(), HVC.getByteTy());
1345 Value *PartM = HVC.insertb(Builder, Zero, HVC.vbytes(Builder, Mask),
1346 S.Seg.Start, S.Seg.Size, S.Pos);
1347 AccumM = Builder.
CreateOr(AccumM, PartM);
1349 Value *PartV = HVC.insertb(Builder, Undef, HVC.vbytes(Builder, Pay),
1350 S.Seg.Start, S.Seg.Size, S.Pos);
1355 ASpanV.Blocks.emplace_back(AccumV, ScLen, Index * ScLen);
1356 ASpanM.Blocks.emplace_back(AccumM, ScLen, Index * ScLen);
1360 dbgs() <<
"ASpanV before vlalign:\n" << ASpanV <<
'\n';
1361 dbgs() <<
"ASpanM before vlalign:\n" << ASpanM <<
'\n';
1366 for (
int Index = 1;
Index != NumSectors + 2; ++
Index) {
1367 Value *PrevV = ASpanV[
Index - 1].Seg.Val, *ThisV = ASpanV[
Index].Seg.Val;
1368 Value *PrevM = ASpanM[
Index - 1].Seg.Val, *ThisM = ASpanM[
Index].Seg.Val;
1370 ASpanV[
Index - 1].Seg.Val = HVC.vlalignb(Builder, PrevV, ThisV, AlignVal);
1371 ASpanM[
Index - 1].Seg.Val = HVC.vlalignb(Builder, PrevM, ThisM, AlignVal);
1376 dbgs() <<
"ASpanV after vlalign:\n" << ASpanV <<
'\n';
1377 dbgs() <<
"ASpanM after vlalign:\n" << ASpanM <<
'\n';
1380 auto createStore = [&](IRBuilderBase &Builder,
const ByteSpan &ASpanV,
1381 const ByteSpan &ASpanM,
int Index,
bool MakePred) {
1384 if (HVC.isUndef(Val) || HVC.isZero(Mask))
1387 createAdjustedPointer(Builder, AlignAddr, SecTy, Index * ScLen);
1389 MakePred ? makeTestIfUnaligned(Builder, AlignVal, ScLen) : nullptr;
1394 int Width = (1 + DoAlign) * ScLen;
1395 this->createStore(Builder, Val,
Ptr, Predicate, ScLen,
1396 HVC.vlsb(Builder, Mask),
1397 VSpan.section(Start, Width).values());
1400 for (
int Index = 0;
Index != NumSectors + DoAlign; ++
Index) {
1401 createStore(Builder, ASpanV, ASpanM, Index, DoAlign && Index == NumSectors);
1405auto AlignVectors::realignGroup(
const MoveGroup &Move)
const ->
bool {
1414 auto getMaxOf = [](
auto Range,
auto GetValue) {
1416 return GetValue(
A) < GetValue(
B);
1420 const AddrList &BaseInfos = AddrGroups.at(Move.Base);
1435 std::set<Instruction *> TestSet(Move.Main.begin(), Move.Main.end());
1438 BaseInfos, std::back_inserter(MoveInfos),
1439 [&TestSet](
const AddrInfo &AI) {
return TestSet.count(AI.Inst); });
1442 const AddrInfo &WithMaxAlign =
1443 getMaxOf(MoveInfos, [](
const AddrInfo &AI) {
return AI.HaveAlign; });
1444 Align MaxGiven = WithMaxAlign.HaveAlign;
1447 const AddrInfo &WithMinOffset =
1448 getMaxOf(MoveInfos, [](
const AddrInfo &AI) {
return -AI.Offset; });
1450 const AddrInfo &WithMaxNeeded =
1451 getMaxOf(MoveInfos, [](
const AddrInfo &AI) {
return AI.NeedAlign; });
1452 Align MinNeeded = WithMaxNeeded.NeedAlign;
1465 InstSimplifyFolder(HVC.DL));
1466 Value *AlignAddr =
nullptr;
1467 Value *AlignVal =
nullptr;
1469 if (MinNeeded <= MaxGiven) {
1470 int Start = WithMinOffset.Offset;
1471 int OffAtMax = WithMaxAlign.Offset;
1478 int Adjust = -
alignTo(OffAtMax - Start, MinNeeded.value());
1479 AlignAddr = createAdjustedPointer(Builder, WithMaxAlign.Addr,
1480 WithMaxAlign.ValTy, Adjust, Move.Clones);
1481 int Diff =
Start - (OffAtMax + Adjust);
1482 AlignVal = HVC.getConstInt(Diff);
1484 assert(
static_cast<decltype(MinNeeded.value())
>(Diff) < MinNeeded.value());
1494 createAlignedPointer(Builder, WithMinOffset.Addr, WithMinOffset.ValTy,
1495 MinNeeded.value(), Move.Clones);
1497 Builder.
CreatePtrToInt(WithMinOffset.Addr, HVC.getIntTy(),
"pti");
1499 for (
auto [Old, New] : Move.Clones)
1500 I->replaceUsesOfWith(Old, New);
1505 for (
const AddrInfo &AI : MoveInfos) {
1506 VSpan.Blocks.emplace_back(AI.Inst, HVC.getSizeOf(AI.ValTy),
1507 AI.Offset - WithMinOffset.Offset);
1514 : std::max<int>(MinNeeded.value(), 4);
1515 assert(!Move.IsHvx || ScLen == 64 || ScLen == 128);
1516 assert(Move.IsHvx || ScLen == 4 || ScLen == 8);
1519 dbgs() <<
"ScLen: " << ScLen <<
"\n";
1520 dbgs() <<
"AlignVal:" << *AlignVal <<
"\n";
1521 dbgs() <<
"AlignAddr:" << *AlignAddr <<
"\n";
1522 dbgs() <<
"VSpan:\n" << VSpan <<
'\n';
1526 realignLoadGroup(Builder, VSpan, ScLen, AlignVal, AlignAddr);
1528 realignStoreGroup(Builder, VSpan, ScLen, AlignVal, AlignAddr);
1530 for (
auto *Inst : Move.Main)
1531 Inst->eraseFromParent();
1536auto AlignVectors::makeTestIfUnaligned(IRBuilderBase &Builder,
Value *AlignVal,
1537 int Alignment)
const ->
Value * {
1538 auto *AlignTy = AlignVal->getType();
1540 AlignVal, ConstantInt::get(AlignTy, Alignment - 1),
"and");
1541 Value *
Zero = ConstantInt::get(AlignTy, 0);
1545auto AlignVectors::isSectorTy(
Type *Ty)
const ->
bool {
1546 if (!HVC.isByteVecTy(Ty))
1548 int Size = HVC.getSizeOf(Ty);
1554auto AlignVectors::run() ->
bool {
1557 if (!createAddressGroups())
1561 dbgs() <<
"Address groups(" << AddrGroups.size() <<
"):\n";
1562 for (
auto &[In, AL] : AddrGroups) {
1563 for (
const AddrInfo &AI : AL)
1564 dbgs() <<
"---\n" << AI <<
'\n';
1569 MoveList LoadGroups, StoreGroups;
1571 for (
auto &
G : AddrGroups) {
1577 dbgs() <<
"\nLoad groups(" << LoadGroups.size() <<
"):\n";
1578 for (
const MoveGroup &
G : LoadGroups)
1579 dbgs() <<
G <<
"\n";
1580 dbgs() <<
"Store groups(" << StoreGroups.size() <<
"):\n";
1581 for (
const MoveGroup &
G : StoreGroups)
1582 dbgs() <<
G <<
"\n";
1586 unsigned CountLimit = VAGroupCountLimit;
1587 if (CountLimit == 0)
1590 if (LoadGroups.size() > CountLimit) {
1591 LoadGroups.resize(CountLimit);
1592 StoreGroups.clear();
1594 unsigned StoreLimit = CountLimit - LoadGroups.size();
1595 if (StoreGroups.size() > StoreLimit)
1596 StoreGroups.resize(StoreLimit);
1599 for (
auto &M : LoadGroups)
1601 for (
auto &M : StoreGroups)
1606 for (
auto &M : LoadGroups)
1608 for (
auto &M : StoreGroups)
1618auto HvxIdioms::getNumSignificantBits(
Value *V, Instruction *In)
const
1619 -> std::pair<unsigned, Signedness> {
1620 unsigned Bits = HVC.getNumSignificantBits(V, In);
1626 KnownBits Known = HVC.getKnownBits(V, In);
1627 Signedness Sign =
Signed;
1628 unsigned NumToTest = 0;
1632 NumToTest =
Bits - 1;
1645 return {
Bits, Sign};
1648auto HvxIdioms::canonSgn(SValue
X, SValue
Y)
const
1649 -> std::pair<SValue, SValue> {
1662auto HvxIdioms::matchFxpMul(Instruction &In)
const -> std::optional<FxpOp> {
1663 using namespace PatternMatch;
1664 auto *Ty =
In.getType();
1667 return std::nullopt;
1676 auto m_Shr = [](
auto &&
V,
auto &&S) {
1688 if (
Op.Frac > Width)
1689 return std::nullopt;
1696 return std::nullopt;
1704 Op.Opcode = Instruction::Mul;
1706 Op.X.Sgn = getNumSignificantBits(
Op.X.Val, &In).second;
1707 Op.Y.Sgn = getNumSignificantBits(
Op.Y.Val, &In).second;
1712 return std::nullopt;
1715auto HvxIdioms::processFxpMul(Instruction &In,
const FxpOp &
Op)
const
1717 assert(
Op.X.Val->getType() ==
Op.Y.Val->getType());
1720 if (VecTy ==
nullptr)
1723 unsigned ElemWidth = ElemTy->getBitWidth();
1726 if ((HVC.length(VecTy) * ElemWidth) % (8 * HVC.HST.
getVectorLength()) != 0)
1736 if (ElemWidth <= 32 &&
Op.Frac == 0)
1739 auto [BitsX, SignX] = getNumSignificantBits(
Op.X.Val, &In);
1740 auto [BitsY, SignY] = getNumSignificantBits(
Op.Y.Val, &In);
1746 InstSimplifyFolder(HVC.DL));
1748 auto roundUpWidth = [](
unsigned Width) ->
unsigned {
1754 if (Width > 32 && Width % 32 != 0) {
1761 BitsX = roundUpWidth(BitsX);
1762 BitsY = roundUpWidth(BitsY);
1767 unsigned Width = std::max(BitsX, BitsY);
1769 auto *ResizeTy = VectorType::get(HVC.getIntTy(Width), VecTy);
1770 if (Width < ElemWidth) {
1773 }
else if (Width > ElemWidth) {
1780 assert(
X->getType() ==
Y->getType() &&
X->getType() == ResizeTy);
1782 unsigned VecLen = HVC.length(ResizeTy);
1783 unsigned ChopLen = (8 * HVC.HST.
getVectorLength()) / std::min(Width, 32u);
1787 ChopOp.ResTy = VectorType::get(
Op.ResTy->getElementType(), ChopLen,
false);
1789 for (
unsigned V = 0;
V != VecLen / ChopLen; ++
V) {
1790 ChopOp.X.Val = HVC.subvector(Builder,
X, V * ChopLen, ChopLen);
1791 ChopOp.Y.Val = HVC.subvector(Builder,
Y, V * ChopLen, ChopLen);
1792 Results.push_back(processFxpMulChopped(Builder, In, ChopOp));
1807auto HvxIdioms::processFxpMulChopped(IRBuilderBase &Builder, Instruction &In,
1808 const FxpOp &
Op)
const ->
Value * {
1809 assert(
Op.X.Val->getType() ==
Op.Y.Val->getType());
1811 unsigned Width = InpTy->getScalarSizeInBits();
1814 if (!
Op.RoundAt || *
Op.RoundAt ==
Op.Frac - 1) {
1817 Value *QMul =
nullptr;
1819 QMul = createMulQ15(Builder,
Op.X,
Op.Y, Rounding);
1820 }
else if (Width == 32) {
1821 QMul = createMulQ31(Builder,
Op.X,
Op.Y, Rounding);
1823 if (QMul !=
nullptr)
1829 assert(Width < 32 || Width % 32 == 0);
1839 assert(
Op.Frac != 0 &&
"Unshifted mul should have been skipped");
1840 if (
Op.Frac == 16) {
1842 if (
Value *MulH = createMulH16(Builder,
Op.X,
Op.Y))
1846 Value *Prod32 = createMul16(Builder,
Op.X,
Op.Y);
1848 Value *RoundVal = HVC.getConstSplat(Prod32->
getType(), 1 << *
Op.RoundAt);
1849 Prod32 = Builder.
CreateAdd(Prod32, RoundVal,
"add");
1854 ? Builder.
CreateAShr(Prod32, ShiftAmt,
"asr")
1855 : Builder.
CreateLShr(Prod32, ShiftAmt,
"lsr");
1856 return Builder.
CreateTrunc(Shifted, InpTy,
"trn");
1863 auto WordX = HVC.splitVectorElements(Builder,
Op.X.Val, 32);
1864 auto WordY = HVC.splitVectorElements(Builder,
Op.Y.Val, 32);
1865 auto WordP = createMulLong(Builder, WordX,
Op.X.Sgn, WordY,
Op.Y.Sgn);
1870 if (
Op.RoundAt.has_value()) {
1873 RoundV[*
Op.RoundAt / 32] =
1874 HVC.getConstSplat(HvxWordTy, 1 << (*
Op.RoundAt % 32));
1875 WordP = createAddLong(Builder, WordP, RoundV);
1881 unsigned SkipWords =
Op.Frac / 32;
1882 Constant *ShiftAmt = HVC.getConstSplat(HvxWordTy,
Op.Frac % 32);
1884 for (
int Dst = 0, End = WordP.size() - SkipWords; Dst != End; ++Dst) {
1885 int Src = Dst + SkipWords;
1887 if (Src + 1 < End) {
1898 WordP.resize(WordP.size() - SkipWords);
1900 return HVC.joinVectorElements(Builder, WordP,
Op.ResTy);
1903auto HvxIdioms::createMulQ15(IRBuilderBase &Builder, SValue
X, SValue
Y,
1904 bool Rounding)
const ->
Value * {
1905 assert(
X.Val->getType() ==
Y.Val->getType());
1906 assert(
X.Val->getType()->getScalarType() == HVC.getIntTy(16));
1913 auto V6_vmpyhvsrs = HVC.HST.
getIntrinsicId(Hexagon::V6_vmpyhvsrs);
1914 return HVC.createHvxIntrinsic(Builder, V6_vmpyhvsrs,
X.Val->getType(),
1918auto HvxIdioms::createMulQ31(IRBuilderBase &Builder, SValue
X, SValue
Y,
1919 bool Rounding)
const ->
Value * {
1920 Type *InpTy =
X.Val->getType();
1921 assert(InpTy ==
Y.Val->getType());
1933 HVC.createHvxIntrinsic(Builder, V6_vmpyewuh, InpTy, {
X.Val,
Y.Val});
1934 return HVC.createHvxIntrinsic(Builder, V6_vmpyo_acc, InpTy,
1935 {V1,
X.Val,
Y.Val});
1938auto HvxIdioms::createAddCarry(IRBuilderBase &Builder,
Value *
X,
Value *
Y,
1939 Value *CarryIn)
const
1940 -> std::pair<Value *, Value *> {
1941 assert(
X->getType() ==
Y->getType());
1950 if (CarryIn ==
nullptr)
1951 CarryIn = HVC.getNullValue(HVC.getBoolTy(HVC.length(VecTy)));
1952 Args.push_back(CarryIn);
1954 Value *
Ret = HVC.createHvxIntrinsic(Builder, AddCarry,
1958 return {
Result, CarryOut};
1965 if (CarryIn !=
nullptr) {
1966 unsigned Width = VecTy->getScalarSizeInBits();
1969 for (
unsigned i = 0, e = 32 / Width; i !=
e; ++i)
1970 Mask = (Mask << Width) | 1;
1974 HVC.createHvxIntrinsic(Builder, V6_vandqrt,
nullptr,
1975 {CarryIn, HVC.getConstInt(Mask)});
1976 Result1 = Builder.
CreateAdd(
X, ValueIn,
"add");
1982 return {Result2, Builder.
CreateOr(CarryOut1, CarryOut2,
"orb")};
1985auto HvxIdioms::createMul16(IRBuilderBase &Builder, SValue
X, SValue
Y)
const
1988 std::tie(
X,
Y) = canonSgn(
X,
Y);
2001 HVC.createHvxIntrinsic(Builder, V6_vmpyh, HvxP32Ty, {
Y.Val,
X.Val});
2003 return HVC.vshuff(Builder, HVC.sublo(Builder,
P), HVC.subhi(Builder,
P));
2006auto HvxIdioms::createMulH16(IRBuilderBase &Builder, SValue
X, SValue
Y)
const
2008 Type *HvxI16Ty = HVC.getHvxTy(HVC.getIntTy(16),
false);
2013 return HVC.createHvxIntrinsic(Builder, V6_vmpyuhvs, HvxI16Ty,
2018 Type *HvxP16Ty = HVC.getHvxTy(HVC.getIntTy(16),
true);
2021 unsigned Len = HVC.length(HvxP16Ty) / 2;
2023 SmallVector<int, 128> PickOdd(Len);
2024 for (
int i = 0; i !=
static_cast<int>(
Len); ++i)
2025 PickOdd[i] = 2 * i + 1;
2028 HVC.sublo(Builder, Pair16), HVC.subhi(Builder, Pair16), PickOdd,
"shf");
2031auto HvxIdioms::createMul32(IRBuilderBase &Builder, SValue
X, SValue
Y)
const
2032 -> std::pair<Value *, Value *> {
2033 assert(
X.Val->getType() ==
Y.Val->getType());
2034 assert(
X.Val->getType() == HvxI32Ty);
2037 std::tie(
X,
Y) = canonSgn(
X,
Y);
2040 V6_vmpy_parts = Intrinsic::hexagon_V6_vmpyss_parts;
2042 V6_vmpy_parts = Intrinsic::hexagon_V6_vmpyus_parts;
2044 V6_vmpy_parts = Intrinsic::hexagon_V6_vmpyuu_parts;
2047 Value *Parts = HVC.createHvxIntrinsic(Builder, V6_vmpy_parts,
nullptr,
2048 {
X.Val,
Y.Val}, {HvxI32Ty});
2057 assert(WordX.size() == WordY.size());
2058 unsigned Idx = 0,
Length = WordX.size();
2062 if (HVC.isZero(WordX[Idx]))
2063 Sum[Idx] = WordY[Idx];
2064 else if (HVC.isZero(WordY[Idx]))
2065 Sum[Idx] = WordX[Idx];
2071 Value *Carry =
nullptr;
2072 for (; Idx !=
Length; ++Idx) {
2073 std::tie(Sum[Idx], Carry) =
2074 createAddCarry(Builder, WordX[Idx], WordY[Idx], Carry);
2088 for (
int i = 0, e = WordX.size(); i != e; ++i) {
2089 for (
int j = 0, f = WordY.size(); j != f; ++j) {
2091 Signedness SX = (i + 1 ==
e) ? SgnX :
Unsigned;
2093 auto [
Lo,
Hi] = createMul32(Builder, {WordX[i], SX}, {WordY[
j],
SY});
2094 Products[i +
j + 0].push_back(
Lo);
2095 Products[i +
j + 1].push_back(
Hi);
2109 for (
int i = 0, e = Products.size(); i != e; ++i) {
2110 while (Products[i].
size() > 1) {
2111 Value *Carry =
nullptr;
2112 for (
int j = i;
j !=
e; ++
j) {
2113 auto &ProdJ = Products[
j];
2114 auto [Sum, CarryOut] = createAddCarry(Builder, pop_back_or_zero(ProdJ),
2115 pop_back_or_zero(ProdJ), Carry);
2116 ProdJ.insert(ProdJ.begin(), Sum);
2123 for (
auto &
P : Products) {
2124 assert(
P.size() == 1 &&
"Should have been added together");
2131auto HvxIdioms::run() ->
bool {
2134 for (BasicBlock &
B : HVC.F) {
2135 for (
auto It =
B.rbegin(); It !=
B.rend(); ++It) {
2136 if (
auto Fxm = matchFxpMul(*It)) {
2137 Value *
New = processFxpMul(*It, *Fxm);
2143 It->replaceAllUsesWith(New);
2145 It = StartOver ?
B.rbegin()
2157auto HexagonVectorCombine::run() ->
bool {
2159 dbgs() <<
"Module before HexagonVectorCombine\n" << *
F.getParent();
2162 if (HST.useHVXOps()) {
2164 Changed |= AlignVectors(*this).run();
2166 Changed |= HvxIdioms(*this).run();
2170 dbgs() <<
"Module " << (
Changed ?
"(modified)" :
"(unchanged)")
2171 <<
" after HexagonVectorCombine\n"
2177auto HexagonVectorCombine::getIntTy(
unsigned Width)
const -> IntegerType * {
2181auto HexagonVectorCombine::getByteTy(
int ElemCount)
const ->
Type * {
2183 IntegerType *ByteTy = Type::getInt8Ty(
F.getContext());
2186 return VectorType::get(ByteTy, ElemCount,
false);
2189auto HexagonVectorCombine::getBoolTy(
int ElemCount)
const ->
Type * {
2191 IntegerType *BoolTy = Type::getInt1Ty(
F.getContext());
2194 return VectorType::get(BoolTy, ElemCount,
false);
2197auto HexagonVectorCombine::getConstInt(
int Val,
unsigned Width)
const
2202auto HexagonVectorCombine::isZero(
const Value *Val)
const ->
bool {
2204 return C->isZeroValue();
2208auto HexagonVectorCombine::getIntValue(
const Value *Val)
const
2209 -> std::optional<APInt> {
2211 return CI->getValue();
2212 return std::nullopt;
2215auto HexagonVectorCombine::isUndef(
const Value *Val)
const ->
bool {
2219auto HexagonVectorCombine::isTrue(
const Value *Val)
const ->
bool {
2223auto HexagonVectorCombine::isFalse(
const Value *Val)
const ->
bool {
2227auto HexagonVectorCombine::getHvxTy(
Type *ElemTy,
bool Pair)
const
2233 "Invalid HVX element type");
2234 unsigned HwLen = HST.getVectorLength();
2236 return VectorType::get(ElemTy, Pair ? 2 * NumElems : NumElems,
2240auto HexagonVectorCombine::getSizeOf(
const Value *Val, SizeKind Kind)
const
2242 return getSizeOf(Val->
getType(), Kind);
2245auto HexagonVectorCombine::getSizeOf(
const Type *Ty, SizeKind Kind)
const
2247 auto *NcTy =
const_cast<Type *
>(Ty);
2250 return DL.getTypeStoreSize(NcTy).getFixedValue();
2252 return DL.getTypeAllocSize(NcTy).getFixedValue();
2257auto HexagonVectorCombine::getTypeAlignment(
Type *Ty)
const ->
int {
2260 if (HST.isTypeForHVX(Ty))
2261 return HST.getVectorLength();
2262 return DL.getABITypeAlign(Ty).value();
2265auto HexagonVectorCombine::length(
Value *Val)
const ->
size_t {
2266 return length(Val->
getType());
2269auto HexagonVectorCombine::length(
Type *Ty)
const ->
size_t {
2271 assert(VecTy &&
"Must be a vector type");
2272 return VecTy->getElementCount().getFixedValue();
2275auto HexagonVectorCombine::getNullValue(
Type *Ty)
const ->
Constant * {
2283auto HexagonVectorCombine::getFullValue(
Type *Ty)
const ->
Constant * {
2291auto HexagonVectorCombine::getConstSplat(
Type *Ty,
int Val)
const
2295 Type *ElemTy = VecTy->getElementType();
2298 ConstantInt::get(ElemTy, Val));
2302auto HexagonVectorCombine::simplify(
Value *V)
const ->
Value * {
2304 SimplifyQuery Q(
DL, &TLI, &DT, &AC, In);
2311auto HexagonVectorCombine::insertb(IRBuilderBase &Builder,
Value *Dst,
2313 int Where)
const ->
Value * {
2314 assert(isByteVecTy(Dst->getType()) && isByteVecTy(Src->getType()));
2315 int SrcLen = getSizeOf(Src);
2316 int DstLen = getSizeOf(Dst);
2322 Value *P2Src = vresize(Builder, Src, P2Len,
Poison);
2323 Value *P2Dst = vresize(Builder, Dst, P2Len,
Poison);
2326 for (
int i = 0; i != P2Len; ++i) {
2330 (Where <= i && i < Where +
Length) ? P2Len + Start + (i - Where) : i;
2334 return vresize(Builder, P2Insert, DstLen,
Poison);
2337auto HexagonVectorCombine::vlalignb(IRBuilderBase &Builder,
Value *
Lo,
2339 assert(
Lo->getType() ==
Hi->getType() &&
"Argument type mismatch");
2342 int VecLen = getSizeOf(
Hi);
2343 if (
auto IntAmt = getIntValue(Amt))
2344 return getElementRange(Builder,
Lo,
Hi, VecLen - IntAmt->getSExtValue(),
2347 if (HST.isTypeForHVX(
Hi->getType())) {
2348 assert(
static_cast<unsigned>(VecLen) == HST.getVectorLength() &&
2349 "Expecting an exact HVX type");
2350 return createHvxIntrinsic(Builder, HST.getIntrinsicId(Hexagon::V6_vlalignb),
2351 Hi->getType(), {Hi, Lo, Amt});
2359 Builder.
CreateTrunc(Shift, Type::getInt32Ty(
F.getContext()),
"trn");
2364 return vralignb(Builder,
Lo,
Hi,
Sub);
2369auto HexagonVectorCombine::vralignb(IRBuilderBase &Builder,
Value *
Lo,
2371 assert(
Lo->getType() ==
Hi->getType() &&
"Argument type mismatch");
2374 int VecLen = getSizeOf(
Lo);
2375 if (
auto IntAmt = getIntValue(Amt))
2376 return getElementRange(Builder,
Lo,
Hi, IntAmt->getSExtValue(), VecLen);
2378 if (HST.isTypeForHVX(
Lo->getType())) {
2379 assert(
static_cast<unsigned>(VecLen) == HST.getVectorLength() &&
2380 "Expecting an exact HVX type");
2381 return createHvxIntrinsic(Builder, HST.getIntrinsicId(Hexagon::V6_valignb),
2382 Lo->getType(), {Hi, Lo, Amt});
2389 Builder.
CreateTrunc(Shift, Type::getInt32Ty(
F.getContext()),
"trn");
2393 Type *Int64Ty = Type::getInt64Ty(
F.getContext());
2405auto HexagonVectorCombine::concat(IRBuilderBase &Builder,
2409 std::vector<Value *> Work[2];
2410 int ThisW = 0, OtherW = 1;
2412 Work[ThisW].assign(Vecs.begin(), Vecs.end());
2413 while (Work[ThisW].
size() > 1) {
2415 SMask.
resize(length(Ty) * 2);
2416 std::iota(SMask.
begin(), SMask.
end(), 0);
2418 Work[OtherW].clear();
2419 if (Work[ThisW].
size() % 2 != 0)
2421 for (
int i = 0, e = Work[ThisW].
size(); i <
e; i += 2) {
2423 Work[ThisW][i], Work[ThisW][i + 1], SMask,
"shf");
2424 Work[OtherW].push_back(Joined);
2432 SMask.
resize(Vecs.size() * length(Vecs.front()->getType()));
2433 std::iota(SMask.
begin(), SMask.
end(), 0);
2438auto HexagonVectorCombine::vresize(IRBuilderBase &Builder,
Value *Val,
2442 assert(ValTy->getElementType() == Pad->getType());
2444 int CurSize = length(ValTy);
2445 if (CurSize == NewSize)
2448 if (CurSize > NewSize)
2449 return getElementRange(Builder, Val, Val, 0, NewSize);
2451 SmallVector<int, 128> SMask(NewSize);
2452 std::iota(SMask.
begin(), SMask.
begin() + CurSize, 0);
2453 std::fill(SMask.
begin() + CurSize, SMask.
end(), CurSize);
2458auto HexagonVectorCombine::rescale(IRBuilderBase &Builder,
Value *Mask,
2465 Type *FromSTy = FromTy->getScalarType();
2466 Type *ToSTy = ToTy->getScalarType();
2467 if (FromSTy == ToSTy)
2470 int FromSize = getSizeOf(FromSTy);
2471 int ToSize = getSizeOf(ToSTy);
2472 assert(FromSize % ToSize == 0 || ToSize % FromSize == 0);
2475 int FromCount = length(MaskTy);
2476 int ToCount = (FromCount * FromSize) / ToSize;
2477 assert((FromCount * FromSize) % ToSize == 0);
2479 auto *FromITy =
getIntTy(FromSize * 8);
2480 auto *ToITy =
getIntTy(ToSize * 8);
2485 Mask, VectorType::get(FromITy, FromCount,
false),
"sxt");
2487 Ext, VectorType::get(ToITy, ToCount,
false),
"cst");
2489 Cast, VectorType::get(getBoolTy(), ToCount,
false),
"trn");
2493auto HexagonVectorCombine::vlsb(IRBuilderBase &Builder,
Value *Val)
const
2496 if (ScalarTy == getBoolTy())
2499 Value *Bytes = vbytes(Builder, Val);
2501 return Builder.
CreateTrunc(Bytes, getBoolTy(getSizeOf(VecTy)),
"trn");
2504 return Builder.
CreateTrunc(Bytes, getBoolTy(),
"trn");
2508auto HexagonVectorCombine::vbytes(IRBuilderBase &Builder,
Value *Val)
const
2511 if (ScalarTy == getByteTy())
2514 if (ScalarTy != getBoolTy())
2515 return Builder.
CreateBitCast(Val, getByteTy(getSizeOf(Val)),
"cst");
2518 return Builder.
CreateSExt(Val, VectorType::get(getByteTy(), VecTy),
"sxt");
2519 return Builder.
CreateSExt(Val, getByteTy(),
"sxt");
2522auto HexagonVectorCombine::subvector(IRBuilderBase &Builder,
Value *Val,
2523 unsigned Start,
unsigned Length)
const
2526 return getElementRange(Builder, Val, Val, Start,
Length);
2529auto HexagonVectorCombine::sublo(IRBuilderBase &Builder,
Value *Val)
const
2531 size_t Len = length(Val);
2532 assert(Len % 2 == 0 &&
"Length should be even");
2533 return subvector(Builder, Val, 0, Len / 2);
2536auto HexagonVectorCombine::subhi(IRBuilderBase &Builder,
Value *Val)
const
2538 size_t Len = length(Val);
2539 assert(Len % 2 == 0 &&
"Length should be even");
2540 return subvector(Builder, Val, Len / 2, Len / 2);
2543auto HexagonVectorCombine::vdeal(IRBuilderBase &Builder,
Value *Val0,
2545 assert(Val0->getType() == Val1->getType());
2546 int Len = length(Val0);
2547 SmallVector<int, 128>
Mask(2 * Len);
2549 for (
int i = 0; i !=
Len; ++i) {
2556auto HexagonVectorCombine::vshuff(IRBuilderBase &Builder,
Value *Val0,
2558 assert(Val0->getType() == Val1->getType());
2559 int Len = length(Val0);
2560 SmallVector<int, 128>
Mask(2 * Len);
2562 for (
int i = 0; i !=
Len; ++i) {
2563 Mask[2 * i + 0] = i;
2569auto HexagonVectorCombine::createHvxIntrinsic(IRBuilderBase &Builder,
2575 auto getCast = [&](IRBuilderBase &Builder,
Value *Val,
2577 Type *SrcTy = Val->getType();
2578 if (SrcTy == DestTy)
2583 assert(HST.isTypeForHVX(SrcTy,
true));
2585 Type *BoolTy = Type::getInt1Ty(
F.getContext());
2590 unsigned HwLen = HST.getVectorLength();
2591 Intrinsic::ID TC = HwLen == 64 ? Intrinsic::hexagon_V6_pred_typecast
2592 : Intrinsic::hexagon_V6_pred_typecast_128B;
2602 for (
int i = 0, e =
Args.size(); i != e; ++i) {
2604 Type *
T = IntrTy->getParamType(i);
2605 if (
A->getType() !=
T) {
2611 StringRef MaybeName = !IntrTy->getReturnType()->isVoidTy() ?
"cup" :
"";
2612 CallInst *
Call = Builder.
CreateCall(IntrFn, IntrArgs, MaybeName);
2619 if (RetTy ==
nullptr || CallTy == RetTy)
2622 assert(HST.isTypeForHVX(CallTy,
true));
2623 return getCast(Builder,
Call, RetTy);
2626auto HexagonVectorCombine::splitVectorElements(IRBuilderBase &Builder,
2628 unsigned ToWidth)
const
2643 assert(VecTy->getElementType()->isIntegerTy());
2644 unsigned FromWidth = VecTy->getScalarSizeInBits();
2646 assert(ToWidth <= FromWidth &&
"Breaking up into wider elements?");
2647 unsigned NumResults = FromWidth / ToWidth;
2651 unsigned Length = length(VecTy);
2655 auto splitInHalf = [&](
unsigned Begin,
unsigned End,
auto splitFunc) ->
void {
2659 if (Begin + 1 == End)
2665 auto *VTy = VectorType::get(
getIntTy(Width / 2), 2 *
Length,
false);
2668 Value *Res =
vdeal(Builder, sublo(Builder, VVal), subhi(Builder, VVal));
2670 unsigned Half = (Begin + End) / 2;
2671 Results[Begin] = sublo(Builder, Res);
2672 Results[Half] = subhi(Builder, Res);
2674 splitFunc(Begin, Half, splitFunc);
2675 splitFunc(Half, End, splitFunc);
2678 splitInHalf(0, NumResults, splitInHalf);
2682auto HexagonVectorCombine::joinVectorElements(IRBuilderBase &Builder,
2684 VectorType *ToType)
const
2686 assert(ToType->getElementType()->isIntegerTy());
2697 unsigned ToWidth = ToType->getScalarSizeInBits();
2698 unsigned Width = Inputs.front()->getType()->getScalarSizeInBits();
2699 assert(Width <= ToWidth);
2701 unsigned Length = length(Inputs.front()->getType());
2703 unsigned NeedInputs = ToWidth / Width;
2704 if (Inputs.size() != NeedInputs) {
2709 Last, getConstSplat(
Last->getType(), Width - 1),
"asr");
2710 Inputs.resize(NeedInputs, Sign);
2713 while (Inputs.size() > 1) {
2716 for (
int i = 0, e = Inputs.size(); i < e; i += 2) {
2717 Value *Res =
vshuff(Builder, Inputs[i], Inputs[i + 1]);
2720 Inputs.resize(Inputs.size() / 2);
2723 assert(Inputs.front()->getType() == ToType);
2724 return Inputs.front();
2727auto HexagonVectorCombine::calculatePointerDifference(
Value *Ptr0,
2729 -> std::optional<int> {
2731 const SCEV *Scev0 = SE.getSCEV(Ptr0);
2732 const SCEV *Scev1 = SE.getSCEV(Ptr1);
2733 const SCEV *ScevDiff = SE.getMinusSCEV(Scev0, Scev1);
2735 APInt
V =
Const->getAPInt();
2736 if (
V.isSignedIntN(8 *
sizeof(
int)))
2737 return static_cast<int>(
V.getSExtValue());
2744 I->eraseFromParent();
2746 SmallVector<Instruction *, 8> ToErase;
2749#define CallBuilder(B, F) \
2752 if (auto *I = dyn_cast<Instruction>(V)) \
2753 B_.ToErase.push_back(I); \
2757 auto Simplify = [
this](
Value *
V) {
2763 auto StripBitCast = [](
Value *
V) {
2765 V =
C->getOperand(0);
2769 Ptr0 = StripBitCast(Ptr0);
2770 Ptr1 = StripBitCast(Ptr1);
2772 return std::nullopt;
2776 if (Gep0->getPointerOperand() != Gep1->getPointerOperand())
2777 return std::nullopt;
2778 if (Gep0->getSourceElementType() != Gep1->getSourceElementType())
2779 return std::nullopt;
2781 Builder
B(Gep0->getParent());
2782 int Scale = getSizeOf(Gep0->getSourceElementType(),
Alloc);
2785 if (Gep0->getNumOperands() != 2 || Gep1->getNumOperands() != 2)
2786 return std::nullopt;
2788 Value *Idx0 = Gep0->getOperand(1);
2789 Value *Idx1 = Gep1->getOperand(1);
2794 return Diff->getSExtValue() * Scale;
2796 KnownBits Known0 = getKnownBits(Idx0, Gep0);
2797 KnownBits Known1 = getKnownBits(Idx1, Gep1);
2800 return std::nullopt;
2808 Diff0 =
C->getSExtValue();
2810 return std::nullopt;
2819 Diff1 =
C->getSExtValue();
2821 return std::nullopt;
2824 return (Diff0 + Diff1) * Scale;
2829auto HexagonVectorCombine::getNumSignificantBits(
const Value *V,
2830 const Instruction *CtxI)
const
2835auto HexagonVectorCombine::getKnownBits(
const Value *V,
2836 const Instruction *CtxI)
const
2841auto HexagonVectorCombine::isSafeToClone(
const Instruction &In)
const ->
bool {
2842 if (
In.mayHaveSideEffects() ||
In.isAtomic() ||
In.isVolatile() ||
2843 In.isFenceLike() ||
In.mayReadOrWriteMemory()) {
2851template <
typename T>
2852auto HexagonVectorCombine::isSafeToMoveBeforeInBB(
const Instruction &In,
2854 const T &IgnoreInsts)
const
2857 [
this](
const Instruction &
I) -> std::optional<MemoryLocation> {
2859 switch (
II->getIntrinsicID()) {
2860 case Intrinsic::masked_load:
2862 case Intrinsic::masked_store:
2878 bool MayWrite =
In.mayWriteToMemory();
2879 auto MaybeLoc = getLocOrNone(In);
2881 auto From =
In.getIterator();
2884 bool MoveUp = (To !=
Block.end() && To->comesBefore(&In));
2886 MoveUp ? std::make_pair(To, From) : std::make_pair(std::next(From), To);
2887 for (
auto It =
Range.first; It !=
Range.second; ++It) {
2888 const Instruction &I = *It;
2889 if (llvm::is_contained(IgnoreInsts, &I))
2892 if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
2893 if (II->getIntrinsicID() == Intrinsic::assume)
2900 if (!CB->hasFnAttr(Attribute::WillReturn))
2902 if (!CB->hasFnAttr(Attribute::NoSync))
2905 if (
I.mayReadOrWriteMemory()) {
2906 auto MaybeLocI = getLocOrNone(I);
2907 if (MayWrite || I.mayWriteToMemory()) {
2908 if (!MaybeLoc || !MaybeLocI)
2910 if (!AA.isNoAlias(*MaybeLoc, *MaybeLocI))
2918auto HexagonVectorCombine::isByteVecTy(
Type *Ty)
const ->
bool {
2920 return VecTy->getElementType() == getByteTy();
2924auto HexagonVectorCombine::getElementRange(IRBuilderBase &Builder,
Value *
Lo,
2928 SmallVector<int, 128> SMask(
Length);
2929 std::iota(SMask.
begin(), SMask.
end(), Start);
2936class HexagonVectorCombineLegacy :
public FunctionPass {
2940 HexagonVectorCombineLegacy() : FunctionPass(
ID) {}
2942 StringRef getPassName()
const override {
return "Hexagon Vector Combine"; }
2944 void getAnalysisUsage(AnalysisUsage &AU)
const override {
2952 FunctionPass::getAnalysisUsage(AU);
2956 if (skipFunction(
F))
2958 AliasAnalysis &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
2959 AssumptionCache &AC =
2960 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
F);
2961 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
2962 ScalarEvolution &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
2963 TargetLibraryInfo &TLI =
2964 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
F);
2965 auto &
TM = getAnalysis<TargetPassConfig>().getTM<HexagonTargetMachine>();
2966 HexagonVectorCombine HVC(
F, AA, AC, DT, SE, TLI, TM);
2972char HexagonVectorCombineLegacy::ID = 0;
2975 "Hexagon Vector Combine",
false,
false)
2986 return new HexagonVectorCombineLegacy();
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Prepare AGPR Alloc
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
static IntegerType * getIntTy(IRBuilderBase &B, const TargetLibraryInfo *TLI)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
#define LLVM_ATTRIBUTE_UNUSED
This file defines the DenseMap class.
static bool runOnFunction(Function &F, bool PostInlining)
static cl::opt< unsigned > SizeLimit("eif-limit", cl::init(6), cl::Hidden, cl::desc("Size limit in Hexagon early if-conversion"))
#define CallBuilder(B, F)
static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
static bool isCandidate(const MachineInstr *MI, Register &DefedReg, Register FrameReg)
static bool isUndef(const MachineInstr &MI)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Remove Loads Into Fake Uses
static ConstantInt * getConstInt(MDNode *MD, unsigned NumOp)
This file defines the SmallVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static SymbolRef::Type getType(const Symbol *Sym)
Target-Independent Code Generator Pass Configuration Options pass.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
AnalysisUsage & addRequired()
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
InstListType::const_iterator const_iterator
InstListType::iterator iterator
Instruction iterators...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
AttributeList getAttributes() const
Return the attributes for this call.
@ ICMP_ULT
unsigned less than
This is the shared class of boolean and integer constants.
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static LLVM_ABI Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
iterator_range< iterator > children()
DomTreeNodeBase< NodeT > * getRootNode()
getRootNode - This returns the entry node for the CFG of the function.
Legacy analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
FunctionPass class - This class is used to implement most global optimizations.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
const BasicBlock & back() const
bool isHVXVectorType(EVT VecTy, bool IncludeBool=false) const
bool useHVXV62Ops() const
bool useHVXV69Ops() const
unsigned getVectorLength() const
bool useHVXV66Ops() const
bool isTypeForHVX(Type *VecTy, bool IncludeBool=false) const
Intrinsic::ID getIntrinsicId(unsigned Opc) const
Common base class shared among various IRBuilders.
LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
LLVM_ABI Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)
const char * getOpcodeName() const
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
An instruction for reading from memory.
bool doesNotAccessMemory() const
Whether this function accesses no memory.
bool onlyAccessesInaccessibleMem() const
Whether this function only (at most) accesses inaccessible memory.
static LLVM_ABI std::optional< MemoryLocation > getOrNone(const Instruction *Inst)
static LLVM_ABI MemoryLocation getForArgument(const CallBase *Call, unsigned ArgIdx, const TargetLibraryInfo *TLI)
Return a location representing a particular argument of a call.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
The main scalar evolution driver.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Provides information about what library functions are available for the current target.
Primary interface to the complete machine description for the target machine.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
Target-Independent Code Generator Pass Configuration Options.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
const ParentTy * getParent() const
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Abstract Attribute helper functions.
Rounding
Possible values of current rounding mode, which is specified in bits 23:22 of FPCR.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ BasicBlock
Various leaf nodes.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
bool match(Val *V, const Pattern &P)
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
@ Undef
Value of the register doesn't matter.
initializer< Ty > init(const Ty &Val)
@ User
could "use" a pointer
friend class Instruction
Iterator for Instructions in a `BasicBlock.
LLVM_ABI Instruction * getTerminator() const
LLVM_ABI Instruction & front() const
This is an optimization pass for GlobalISel generic memory operations.
FunctionPass * createHexagonVectorCombineLegacyPass()
FunctionAddr VTableAddr Value
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
LLVM_ABI bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
If the specified value is a trivially dead instruction, delete it.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
MemoryEffectsBase< IRMemLocation > MemoryEffects
Summary of how a function affects memory in the program.
LLVM_ABI Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,...
OutputIt copy_if(R &&Range, OutputIt Out, UnaryPredicate P)
Provide wrappers to std::copy_if which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
detail::concat_range< ValueT, RangeTs... > concat(RangeTs &&...Ranges)
Returns a concatenated range across two or more ranges.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
LLVM_ABI Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
DomTreeNodeBase< BasicBlock > DomTreeNode
auto reverse(ContainerTy &&C)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
DWARFExpression::Operation Op
auto max_element(R &&Range)
Provide wrappers to std::max_element which take ranges instead of having to pass begin/end explicitly...
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
LLVM_ABI unsigned ComputeMaxSignificantBits(const Value *Op, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Get the upper bound on bit size for this Value Op as a signed integer.
AAResults AliasAnalysis
Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.
LLVM_ABI bool mayHaveNonDefUseDependency(const Instruction &I)
Returns true if the result or effects of the given instructions I depend values not reachable through...
MaskT vshuff(ArrayRef< int > Vu, ArrayRef< int > Vv, unsigned Size, bool TakeOdd)
MaskT vdeal(ArrayRef< int > Vu, ArrayRef< int > Vv, unsigned Size, bool TakeOdd)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.