69#define DEBUG_TYPE "hexagon-load-store-widening"
73 cl::desc(
"Limit block size to analyze in load/store widening pass"));
77struct HexagonLoadStoreWidening {
78 enum WideningMode { Store, Load };
79 const HexagonInstrInfo *TII;
80 const HexagonRegisterInfo *TRI;
81 MachineRegisterInfo *MRI;
86 HexagonLoadStoreWidening(
const HexagonInstrInfo *TII,
87 const HexagonRegisterInfo *TRI,
89 MachineFunction *MF,
bool StoreMode)
90 : TII(TII), TRI(TRI), MRI(MRI), AA(AA), MF(MF),
91 Mode(StoreMode ? WideningMode::Store : WideningMode::Load),
92 HII(MF->getSubtarget<HexagonSubtarget>().getInstrInfo()) {}
98 const unsigned MaxWideSize = 8;
99 const HexagonInstrInfo *HII =
nullptr;
101 using InstrSet = SmallPtrSet<MachineInstr *, 16>;
102 using InstrGroup = SmallVector<MachineInstr *, 8>;
103 using InstrGroupList = SmallVector<InstrGroup, 8>;
105 InstrSet ProcessedInsts;
107 unsigned getBaseAddressRegister(
const MachineInstr *
MI);
109 int64_t getPostIncrementValue(
const MachineInstr *
MI);
110 bool handledInstType(
const MachineInstr *
MI);
112 void createGroup(MachineInstr *BaseInst, InstrGroup &Group);
113 void createGroups(MachineBasicBlock &
MBB, InstrGroupList &StoreGroups);
114 bool processBasicBlock(MachineBasicBlock &
MBB);
115 bool processGroup(InstrGroup &Group);
117 InstrGroup &OG,
unsigned &TotalSize,
unsigned MaxSize);
118 bool createWideInsts(InstrGroup &OG, InstrGroup &NG,
unsigned TotalSize);
119 bool createWideStores(InstrGroup &OG, InstrGroup &NG,
unsigned TotalSize);
120 bool createWideLoads(InstrGroup &OG, InstrGroup &NG,
unsigned TotalSize);
121 bool replaceInsts(InstrGroup &OG, InstrGroup &NG);
122 bool areAdjacent(
const MachineInstr *
S1,
const MachineInstr *S2);
123 bool canSwapInstructions(
const MachineInstr *
A,
const MachineInstr *
B);
129 HexagonStoreWidening() : MachineFunctionPass(ID) {}
131 StringRef getPassName()
const override {
return "Hexagon Store Widening"; }
133 void getAnalysisUsage(AnalysisUsage &AU)
const override {
139 bool runOnMachineFunction(MachineFunction &MFn)
override {
144 const HexagonInstrInfo *
TII =
ST.getInstrInfo();
145 const HexagonRegisterInfo *
TRI =
ST.getRegisterInfo();
147 AliasAnalysis *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
149 return HexagonLoadStoreWidening(
TII,
TRI,
MRI, AA, &MFn,
true).run();
156 HexagonLoadWidening() : MachineFunctionPass(ID) {}
158 StringRef getPassName()
const override {
return "Hexagon Load Widening"; }
160 void getAnalysisUsage(AnalysisUsage &AU)
const override {
166 bool runOnMachineFunction(MachineFunction &MFn)
override {
171 const HexagonInstrInfo *
TII =
ST.getInstrInfo();
172 const HexagonRegisterInfo *
TRI =
ST.getRegisterInfo();
174 AliasAnalysis *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
175 return HexagonLoadStoreWidening(
TII,
TRI,
MRI, AA, &MFn,
false).run();
179char HexagonStoreWidening::ID = 0;
180char HexagonLoadWidening::ID = 0;
185 "Hexagon Store Widening",
false,
false)
197 assert(!
MI->memoperands_empty() &&
"Expecting memory operands");
198 return **
MI->memoperands_begin();
202HexagonLoadStoreWidening::getBaseAddressRegister(
const MachineInstr *
MI) {
203 assert(HexagonLoadStoreWidening::handledInstType(
MI) &&
"Unhandled opcode");
206 const MachineOperand &MO =
MI->getOperand(
Base);
207 assert(MO.
isReg() &&
"Expecting register operand");
211int64_t HexagonLoadStoreWidening::getOffset(
const MachineInstr *
MI) {
212 assert(HexagonLoadStoreWidening::handledInstType(
MI) &&
"Unhandled opcode");
222 const MachineOperand &MO =
MI->getOperand(
Offset);
235HexagonLoadStoreWidening::getPostIncrementValue(
const MachineInstr *
MI) {
236 unsigned Base, PostIncIdx;
238 const MachineOperand &MO =
MI->getOperand(PostIncIdx);
244inline bool HexagonLoadStoreWidening::handledInstType(
const MachineInstr *
MI) {
245 unsigned Opc =
MI->getOpcode();
246 if (
Mode == WideningMode::Store) {
248 case Hexagon::S4_storeirb_io:
249 case Hexagon::S4_storeirh_io:
250 case Hexagon::S4_storeiri_io:
251 case Hexagon::S2_storeri_io:
253 return MI->getOperand(0).isReg();
254 case Hexagon::S2_storeri_pi:
255 return MI->getOperand(1).isReg();
262 case Hexagon::L2_loadri_io:
264 return !
MI->memoperands_empty() &&
MI->getOperand(1).isReg() &&
265 MI->getOperand(2).isImm();
266 case Hexagon::L2_loadri_pi:
267 return !
MI->memoperands_empty() &&
MI->getOperand(2).isReg();
276 for (
const auto &
Op :
MI->operands()) {
286bool HexagonLoadStoreWidening::canSwapInstructions(
const MachineInstr *
A,
287 const MachineInstr *
B) {
288 DenseSet<Register> ARegDefs;
289 DenseSet<Register> ARegUses;
291 if (
A->mayLoadOrStore() &&
B->mayLoadOrStore() &&
292 (
A->mayStore() ||
B->mayStore()) &&
A->mayAlias(AA, *
B,
true))
294 for (
const auto &BOp :
B->operands()) {
297 if ((BOp.isDef() || BOp.readsReg()) && ARegDefs.
contains(BOp.getReg()))
299 if (BOp.isDef() && ARegUses.
contains(BOp.getReg()))
314void HexagonLoadStoreWidening::createGroups(MachineBasicBlock &
MBB,
315 InstrGroupList &StoreGroups) {
320 MachineInstr *
MI = &(*I);
321 if (!handledInstType(
MI))
330 StoreGroups.push_back(
G);
337void HexagonLoadStoreWidening::createGroup(MachineInstr *BaseInst,
339 assert(handledInstType(BaseInst) &&
"Unexpected instruction");
340 unsigned BaseReg = getBaseAddressRegister(BaseInst);
352 MachineInstr *
MI = &(*I);
355 if (
MI->isCall() ||
MI->hasUnmodeledSideEffects() ||
356 MI->hasOrderedMemoryRef())
359 if (!handledInstType(
MI)) {
360 if (
MI->mayLoadOrStore())
368 for (
auto GI : Group)
369 if (GI->mayAlias(AA, *
MI,
true))
371 if (
Mode == WideningMode::Load) {
375 for (
auto MemI :
Other)
376 if (!canSwapInstructions(
MI, MemI))
382 for (
auto MemI :
Other) {
383 if (std::distance(Group.back()->getIterator(), MemI->getIterator()) <=
386 for (
auto GI : Group)
387 if (!canSwapInstructions(MemI, GI))
392 unsigned BR = getBaseAddressRegister(
MI);
403bool HexagonLoadStoreWidening::areAdjacent(
const MachineInstr *
S1,
404 const MachineInstr *S2) {
405 if (!handledInstType(
S1) || !handledInstType(S2))
414 return (Off1 >= 0) ? Off1 + S1MO.
getSize().
getValue() == unsigned(Off2)
415 : int(Off1 + S1MO.
getSize().getValue()) == Off2;
424bool HexagonLoadStoreWidening::selectInsts(InstrGroup::iterator Begin,
425 InstrGroup::iterator End,
426 InstrGroup &OG,
unsigned &TotalSize,
428 assert(Begin != End &&
"No instructions to analyze");
429 assert(OG.empty() &&
"Old group not empty on entry");
431 if (std::distance(Begin, End) <= 1)
434 MachineInstr *FirstMI = *Begin;
436 const MachineMemOperand &FirstMMO =
getMemTarget(FirstMI);
442 unsigned FirstOffset =
getOffset(FirstMI);
448 if (SizeAccum >= MaxSize)
453 if (SizeAccum >= Alignment) {
455 dbgs() <<
"Size of load/store greater than equal to its alignment\n");
465 unsigned OffsetOrIncVal = 0;
467 OffsetOrIncVal = getPostIncrementValue(FirstMI);
469 OffsetOrIncVal = FirstOffset;
470 if ((2 * SizeAccum - 1) & OffsetOrIncVal) {
471 LLVM_DEBUG(
dbgs() <<
"Instruction cannot be widened as the offset/postinc"
472 <<
" value: " << getPostIncrementValue(FirstMI)
473 <<
" is invalid in the widened version\n");
477 OG.push_back(FirstMI);
478 MachineInstr *
S1 = FirstMI;
482 unsigned Pow2Num = 1;
483 unsigned Pow2Size = SizeAccum;
491 for (InstrGroup::iterator
I = Begin + 1;
I != End; ++
I) {
492 MachineInstr *S2 = *
I;
495 if (!areAdjacent(
S1, S2))
504 if (SizeAccum + S2Size > std::min(MaxSize, Alignment))
511 Pow2Size = SizeAccum;
513 if ((2 * Pow2Size - 1) & FirstOffset)
527 TotalSize = Pow2Size;
533bool HexagonLoadStoreWidening::createWideInsts(InstrGroup &OG, InstrGroup &NG,
534 unsigned TotalSize) {
535 if (
Mode == WideningMode::Store) {
536 return createWideStores(OG, NG, TotalSize);
538 return createWideLoads(OG, NG, TotalSize);
544bool HexagonLoadStoreWidening::createWideStores(InstrGroup &OG, InstrGroup &NG,
545 unsigned TotalSize) {
550 if (TotalSize > MaxWideSize)
555 bool HaveImm =
false;
556 bool HaveReg =
false;
558 for (MachineInstr *
MI : OG) {
573 Mask = (0xFFFFFFFFFFFFFFFFU >> (64 - NBits));
574 Val = (SO.
getImm() & Mask) << Shift;
587 if (HaveImm && HaveReg) {
588 LLVM_DEBUG(
dbgs() <<
"Cannot merge store register and store imm\n");
592 MachineInstr *FirstSt = OG.front();
595 MachineMemOperand *NewM =
602 auto SecondSt = OG.back();
604 MachineOperand FReg =
610 MachineOperand
SReg = SecondSt->getOperand(2);
612 "Cannot merge store register and store imm");
613 const MCInstrDesc &CombD =
TII->get(Hexagon::A2_combinew);
616 MachineInstr *CombI =
BuildMI(*MF,
DL, CombD, VReg).
add(SReg).
add(FReg);
619 if (FirstSt->
getOpcode() == Hexagon::S2_storeri_pi) {
620 const MCInstrDesc &StD =
TII->get(Hexagon::S2_storerd_pi);
626 const MCInstrDesc &StD =
TII->get(Hexagon::S2_storerd_io);
639 if (TotalSize == 8) {
641 uint64_t
Mask = 0xFFFFFFFFU;
642 int LowerAcc = int(Mask & Acc);
643 int UpperAcc = Acc >> 32;
648 const MCInstrDesc &TfrD =
TII->get(Hexagon::A2_tfrsi);
649 const TargetRegisterClass *RC =
TII->getRegClass(TfrD, 0,
TRI);
653 const MCInstrDesc &CombD =
TII->get(Hexagon::A4_combineir);
660 const MCInstrDesc &CombD =
TII->get(Hexagon::A4_combineii);
664 const MCInstrDesc &StD =
TII->get(Hexagon::S2_storerd_io);
667 }
else if (Acc < 0x10000) {
669 unsigned WOpc = (TotalSize == 2) ? Hexagon::S4_storeirh_io
670 : (TotalSize == 4) ? Hexagon::S4_storeiri_io
672 assert(WOpc &&
"Unexpected size");
674 int Val = (TotalSize == 2) ? int16_t(Acc) : int(Acc);
675 const MCInstrDesc &StD =
TII->get(WOpc);
679 const MCInstrDesc &TfrD =
TII->get(Hexagon::A2_tfrsi);
680 const TargetRegisterClass *RC =
TII->getRegClass(TfrD, 0,
TRI);
685 unsigned WOpc = (TotalSize == 2) ? Hexagon::S2_storerh_io
686 : (TotalSize == 4) ? Hexagon::S2_storeri_io
688 assert(WOpc &&
"Unexpected size");
690 const MCInstrDesc &StD =
TII->get(WOpc);
703bool HexagonLoadStoreWidening::createWideLoads(InstrGroup &OG, InstrGroup &NG,
704 unsigned TotalSize) {
709 if (TotalSize > MaxWideSize)
711 assert(OG.size() == 2 &&
"Expecting two elements in Instruction Group.");
713 MachineInstr *FirstLd = OG.front();
715 MachineMemOperand *NewM =
720 MachineOperand &MRBase =
726 Register NewMR =
MRI->createVirtualRegister(&Hexagon::DoubleRegsRegClass);
730 if (FirstLd->
getOpcode() == Hexagon::L2_loadri_pi) {
749 auto getHalfReg = [&](MachineInstr *DoubleReg,
unsigned SubReg,
750 MachineInstr *DstReg) {
751 Register DestReg = DstReg->getOperand(0).getReg();
752 return BuildMI(*MF,
DL,
TII->get(Hexagon::COPY), DestReg)
756 MachineInstr *LdI_lo = getHalfReg(LdI, Hexagon::isub_lo, FirstLd);
757 MachineInstr *LdI_hi = getHalfReg(LdI, Hexagon::isub_hi, OG.back());
758 NG.push_back(LdI_lo);
759 NG.push_back(LdI_hi);
769bool HexagonLoadStoreWidening::replaceInsts(InstrGroup &OG, InstrGroup &NG) {
771 dbgs() <<
"Replacing:\n";
793 if (
Mode == WideningMode::Load) {
795 for (
auto &
I : *
MBB) {
796 if (OldMemInsts.count(&
I)) {
802 assert((InsertAt !=
MBB->
end()) &&
"Cannot locate any load from the group");
810 if (OldMemInsts.count(&(*
I))) {
811 InsertAt = (*I).getIterator();
816 assert((
I !=
MBB->
rend()) &&
"Cannot locate any store from the group");
818 for (
auto I = NG.rbegin();
I != NG.rend(); ++
I)
823 I->eraseFromParent();
831bool HexagonLoadStoreWidening::processGroup(InstrGroup &Group) {
833 InstrGroup::iterator
I = Group.begin(),
E = Group.end();
835 unsigned CollectedSize;
841 bool Succ = selectInsts(
I++,
E, OG, CollectedSize, MaxWideSize) &&
842 createWideInsts(OG, NG, CollectedSize) && replaceInsts(OG, NG);
846 assert(OG.size() > 1 &&
"Created invalid group");
847 assert(std::distance(
I,
E) + 1 >=
int(OG.size()) &&
"Too many elements");
862bool HexagonLoadStoreWidening::processBasicBlock(MachineBasicBlock &
MBB) {
870 createGroups(
MBB, SGs);
872 auto Less = [
this](
const MachineInstr *
A,
const MachineInstr *
B) ->
bool {
875 for (
auto &
G : SGs) {
876 assert(
G.size() > 1 &&
"Group with fewer than 2 elements");
885bool HexagonLoadStoreWidening::run() {
895 return new HexagonStoreWidening();
899 return new HexagonLoadWidening();
unsigned const MachineRegisterInfo * MRI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
const HexagonInstrInfo * TII
static cl::opt< unsigned > MaxMBBSizeForLoadStoreWidening("max-bb-size-for-load-store-widening", cl::Hidden, cl::init(1000), cl::desc("Limit block size to analyze in load/store widening pass"))
hexagon widen Hexagon Store false hexagon widen Hexagon Load static false const MachineMemOperand & getMemTarget(const MachineInstr *MI)
Register const TargetRegisterInfo * TRI
Promote Memory to Register
This file provides utility analysis objects describing memory locations.
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
static void addDefsUsesToList(const MachineInstr &MI, DenseSet< Register > &RegDefs, DenseSet< Register > &RegUses)
This file defines the SmallPtrSet class.
static unsigned getSize(unsigned Kind)
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Implements a dense probed hash-table based set.
FunctionPass class - This class is used to implement most global optimizations.
bool isPostIncrement(const MachineInstr &MI) const override
Return true for post-incremented instructions.
bool getBaseAndOffsetPosition(const MachineInstr &MI, unsigned &BasePos, unsigned &OffsetPos) const override
For instructions with a base and offset, return the position of the base register and offset operands...
constexpr bool isValid() const
TypeSize getValue() const
LLVM_ABI instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
reverse_iterator rbegin()
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
MachineInstrBundleIterator< MachineInstr > iterator
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
bool memoperands_empty() const
Return true if we don't have any memory operands which described the memory access done by this instr...
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
LLVM_ABI void dump() const
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
A description of a memory reference used in the backend.
LocationSize getSize() const
Return the size in bytes of the memory reference.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
LLVM_ABI Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
LocationSize getSizeInBits() const
Return the size in bits of the memory reference.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
unsigned getSubReg() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
@ MO_Immediate
Immediate operand.
@ MO_GlobalAddress
Address of a global value.
@ MO_Register
Register operand.
int64_t getOffset() const
Return the offset from the symbol in this operand.
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
void push_back(const T &Elt)
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Kill
The last use of a register.
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr from_range_t from_range
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FunctionPass * createHexagonLoadWidening()
unsigned getKillRegState(bool B)
DWARFExpression::Operation Op
AAResults AliasAnalysis
Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.
FunctionPass * createHexagonStoreWidening()
uint64_t value() const
This is a hole in the type system and should not be abused.