38 #define DEBUG_TYPE "arm-cp-islands"
40 STATISTIC(NumCPEs,
"Number of constpool entries");
41 STATISTIC(NumSplit,
"Number of uncond branches inserted");
42 STATISTIC(NumCBrFixed,
"Number of cond branches fixed");
43 STATISTIC(NumUBrFixed,
"Number of uncond branches fixed");
44 STATISTIC(NumTBs,
"Number of table branches generated");
45 STATISTIC(NumT2CPShrunk,
"Number of Thumb2 constantpool instructions shrunk");
46 STATISTIC(NumT2BrShrunk,
"Number of Thumb2 immediate branches shrunk");
47 STATISTIC(NumCBZ,
"Number of CBZ / CBNZ formed");
48 STATISTIC(NumJTMoved,
"Number of jump table destination blocks moved");
49 STATISTIC(NumJTInserted,
"Number of jump table intermediate blocks inserted");
54 cl::desc(
"Adjust basic block layout to better use TB[BH]"));
62 static inline unsigned UnknownPadding(
unsigned LogAlign,
unsigned KnownBits) {
63 if (KnownBits < LogAlign)
64 return (1u << LogAlign) - (1u << KnownBits);
83 struct BasicBlockInfo {
116 BasicBlockInfo() : Offset(0), Size(0), KnownBits(0), Unalign(0),
122 unsigned internalKnownBits()
const {
123 unsigned Bits = Unalign ? Unalign : KnownBits;
126 if (Size & ((1u << Bits) - 1))
134 unsigned postOffset(
unsigned LogAlign = 0)
const {
135 unsigned PO = Offset + Size;
136 unsigned LA = std::max(
unsigned(PostAlign), LogAlign);
148 unsigned postKnownBits(
unsigned LogAlign = 0)
const {
149 return std::max(std::max(
unsigned(PostAlign), LogAlign),
150 internalKnownBits());
154 std::vector<BasicBlockInfo> BBInfo;
159 std::vector<MachineBasicBlock*> WaterList;
165 typedef std::vector<MachineBasicBlock*>::iterator water_iterator;
188 bool neg,
bool soimm)
189 :
MI(mi), CPEMI(cpemi), MaxDisp(maxdisp), NegOk(neg), IsSoImm(soimm),
190 KnownAlignment(
false) {
191 HighWaterMark = CPEMI->getParent();
196 unsigned getMaxDisp()
const {
197 return (KnownAlignment ? MaxDisp : MaxDisp - 2) - 2;
203 std::vector<CPUser> CPUsers;
213 : CPEMI(cpemi), CPI(cpi), RefCount(
rc) {}
225 std::vector<std::vector<CPEntry> > CPEntries;
241 unsigned MaxDisp : 31;
244 ImmBranch(
MachineInstr *mi,
unsigned maxdisp,
bool cond,
unsigned ubr)
245 :
MI(mi), MaxDisp(maxdisp), isCond(cond), UncondBr(ubr) {}
250 std::vector<ImmBranch> ImmBranches;
277 const char *getPassName()
const override {
278 return "ARM constant island placement and branch shortening pass";
282 void doInitialConstPlacement(std::vector<MachineInstr *> &CPEMIs);
283 void doInitialJumpTablePlacement(std::vector<MachineInstr *> &CPEMIs);
285 CPEntry *findConstPoolEntry(
unsigned CPI,
const MachineInstr *CPEMI);
287 void scanFunctionJumpTables();
288 void initializeFunctionInfo(
const std::vector<MachineInstr*> &CPEMIs);
292 bool decrementCPEReferenceCount(
unsigned CPI,
MachineInstr* CPEMI);
294 int findInRangeCPEntry(CPUser& U,
unsigned UserOffset);
295 bool findAvailableWater(CPUser&U,
unsigned UserOffset,
296 water_iterator &WaterIter);
297 void createNewWater(
unsigned CPUserIndex,
unsigned UserOffset,
299 bool handleConstantPoolUser(
unsigned CPUserIndex);
301 bool removeUnusedCPEntries();
304 bool DoDump =
false);
306 CPUser &U,
unsigned &Growth);
308 bool fixupImmediateBr(ImmBranch &Br);
309 bool fixupConditionalBr(ImmBranch &Br);
310 bool fixupUnconditionalBr(ImmBranch &Br);
311 bool undoLRSpillRestore();
313 bool optimizeThumb2Instructions();
314 bool optimizeThumb2Branches();
315 bool reorderThumb2JumpTables();
317 unsigned &DeadSize,
bool &CanDeleteLEA,
319 bool optimizeThumb2JumpTables();
325 unsigned getUserOffset(CPUser&)
const;
329 bool isOffsetInRange(
unsigned UserOffset,
unsigned TrialOffset,
330 unsigned Disp,
bool NegativeOK,
bool IsSoImm =
false);
331 bool isOffsetInRange(
unsigned UserOffset,
unsigned TrialOffset,
333 return isOffsetInRange(UserOffset, TrialOffset,
334 U.getMaxDisp(), U.NegOk, U.IsSoImm);
347 assert(!MBBId || BBInfo[MBBId - 1].postOffset() <= BBInfo[MBBId].Offset);
349 DEBUG(
dbgs() <<
"Verifying " << CPUsers.size() <<
" CP users.\n");
350 for (
unsigned i = 0, e = CPUsers.size(); i != e; ++i) {
351 CPUser &U = CPUsers[i];
352 unsigned UserOffset = getUserOffset(U);
355 if (isCPEntryInRange(U.MI, UserOffset, U.CPEMI, U.getMaxDisp()+2, U.NegOk,
369 void ARMConstantIslands::dumpBBs() {
371 for (
unsigned J = 0, E = BBInfo.size(); J !=E; ++J) {
372 const BasicBlockInfo &BBI = BBInfo[J];
373 dbgs() <<
format(
"%08x BB#%u\t", BBI.Offset, J)
374 <<
" kb=" <<
unsigned(BBI.KnownBits)
376 <<
" pa=" <<
unsigned(BBI.PostAlign)
377 <<
format(
" size=%#x\n", BBInfo[J].Size);
385 return new ARMConstantIslands();
392 DEBUG(
dbgs() <<
"***** ARMConstantIslands: "
393 << MCP->getConstants().size() <<
" CP entries, aligned to "
394 << MCP->getConstantPoolAlignment() <<
" bytes *****\n");
396 STI = &
static_cast<const ARMSubtarget &
>(MF->getSubtarget());
397 TII = STI->getInstrInfo();
400 isThumb = AFI->isThumbFunction();
401 isThumb1 = AFI->isThumb1OnlyFunction();
402 isThumb2 = AFI->isThumb2Function();
407 MF->getRegInfo().invalidateLiveness();
411 MF->RenumberBlocks();
415 bool MadeChange =
false;
417 scanFunctionJumpTables();
418 MadeChange |= reorderThumb2JumpTables();
420 T2JumpTables.clear();
422 MF->RenumberBlocks();
427 std::vector<MachineInstr*> CPEMIs;
429 doInitialConstPlacement(CPEMIs);
431 if (MF->getJumpTableInfo())
432 doInitialJumpTablePlacement(CPEMIs);
435 AFI->initPICLabelUId(CPEMIs.size());
440 initializeFunctionInfo(CPEMIs);
446 if (!T2JumpTables.empty())
447 MF->ensureAlignment(2);
450 MadeChange |= removeUnusedCPEntries();
454 unsigned NoCPIters = 0, NoBRIters = 0;
456 DEBUG(
dbgs() <<
"Beginning CP iteration #" << NoCPIters <<
'\n');
457 bool CPChange =
false;
458 for (
unsigned i = 0, e = CPUsers.size(); i != e; ++i)
459 CPChange |= handleConstantPoolUser(i);
460 if (CPChange && ++NoCPIters > 30)
466 NewWaterList.clear();
468 DEBUG(
dbgs() <<
"Beginning BR iteration #" << NoBRIters <<
'\n');
469 bool BRChange =
false;
470 for (
unsigned i = 0, e = ImmBranches.size(); i != e; ++i)
471 BRChange |= fixupImmediateBr(ImmBranches[i]);
472 if (BRChange && ++NoBRIters > 30)
476 if (!CPChange && !BRChange)
482 if (isThumb2 && !STI->prefers32BitThumb())
483 MadeChange |= optimizeThumb2Instructions();
491 MadeChange |= undoLRSpillRestore();
494 for (
unsigned i = 0, e = CPEntries.size(); i != e; ++i) {
495 for (
unsigned j = 0, je = CPEntries[i].
size(); j != je; ++j) {
496 const CPEntry & CPE = CPEntries[i][j];
497 if (CPE.CPEMI && CPE.CPEMI->getOperand(1).isCPI())
498 AFI->recordCPEClone(i, CPE.CPI);
508 JumpTableEntryIndices.clear();
509 JumpTableUserIndices.clear();
512 T2JumpTables.clear();
520 ARMConstantIslands::doInitialConstPlacement(std::vector<MachineInstr*> &CPEMIs) {
526 unsigned MaxAlign =
Log2_32(MCP->getConstantPoolAlignment());
543 const std::vector<MachineConstantPoolEntry> &CPs = MCP->getConstants();
545 const DataLayout &TD = *MF->getTarget().getDataLayout();
546 for (
unsigned i = 0, e = CPs.
size(); i != e; ++i) {
548 assert(Size >= 4 &&
"Too small constant pool entry");
549 unsigned Align = CPs[i].getAlignment();
553 assert((Size % Align) == 0 &&
"CP Entry not multiple of 4 bytes!");
556 unsigned LogAlign =
Log2_32(Align);
561 CPEMIs.push_back(CPEMI);
565 for (
unsigned a = LogAlign + 1; a <= MaxAlign; ++a)
566 if (InsPoint[a] == InsAt)
570 CPEntries.emplace_back(1, CPEntry(CPEMI, i));
572 DEBUG(
dbgs() <<
"Moved CPI#" << i <<
" to end of function, size = "
573 << Size <<
", align = " << Align <<
'\n');
583 void ARMConstantIslands::doInitialJumpTablePlacement(
584 std::vector<MachineInstr *> &CPEMIs) {
585 unsigned i = CPEntries.size();
586 auto MJTI = MF->getJumpTableInfo();
587 const std::vector<MachineJumpTableEntry> &
JT = MJTI->getJumpTables();
594 switch (
MI->getOpcode()) {
601 JTOpcode = ARM::JUMPTABLE_ADDRS;
604 JTOpcode = ARM::JUMPTABLE_INSTS;
607 JTOpcode = ARM::JUMPTABLE_TBB;
610 JTOpcode = ARM::JUMPTABLE_TBH;
614 unsigned NumOps =
MI->getDesc().getNumOperands();
616 MI->getOperand(NumOps - (
MI->isPredicable() ? 2 : 1));
618 unsigned Size = JT[JTI].MBBs.size() *
sizeof(uint32_t);
626 CPEMIs.push_back(CPEMI);
627 CPEntries.emplace_back(1, CPEntry(CPEMI, JTI));
628 JumpTableEntryIndices.insert(std::make_pair(JTI, CPEntries.size() - 1));
629 if (!LastCorrectlyNumberedBB)
630 LastCorrectlyNumberedBB = &MBB;
634 if (LastCorrectlyNumberedBB)
635 MF->RenumberBlocks(LastCorrectlyNumberedBB);
656 return TooDifficult || FBB ==
nullptr;
661 ARMConstantIslands::CPEntry
662 *ARMConstantIslands::findConstPoolEntry(
unsigned CPI,
664 std::vector<CPEntry> &CPEs = CPEntries[CPI];
667 for (
unsigned i = 0, e = CPEs.size(); i != e; ++i) {
668 if (CPEs[i].CPEMI == CPEMI)
676 unsigned ARMConstantIslands::getCPELogAlign(
const MachineInstr *CPEMI) {
678 case ARM::CONSTPOOL_ENTRY:
680 case ARM::JUMPTABLE_TBB:
682 case ARM::JUMPTABLE_TBH:
683 case ARM::JUMPTABLE_INSTS:
685 case ARM::JUMPTABLE_ADDRS:
691 unsigned CPI = getCombinedIndex(CPEMI);
692 assert(CPI < MCP->getConstants().
size() &&
"Invalid constant pool index.");
693 unsigned Align = MCP->getConstants()[CPI].getAlignment();
701 void ARMConstantIslands::scanFunctionJumpTables() {
708 if (
I->isBranch() &&
I->getOpcode() == ARM::t2BR_JT)
716 void ARMConstantIslands::
717 initializeFunctionInfo(
const std::vector<MachineInstr*> &CPEMIs) {
719 BBInfo.resize(MF->getNumBlockIDs());
730 BBInfo.front().KnownBits = MF->getAlignment();
733 adjustBBOffsetsAfter(MF->begin());
747 if (
I->isDebugValue())
750 unsigned Opc =
I->getOpcode();
760 T2JumpTables.push_back(
I);
793 unsigned MaxOffs = ((1 << (Bits-1))-1) * Scale;
794 ImmBranches.push_back(ImmBranch(
I, MaxOffs, isCond, UOpc));
797 if (Opc == ARM::tPUSH || Opc == ARM::tPOP_RET)
798 PushPopMIs.push_back(
I);
800 if (Opc == ARM::CONSTPOOL_ENTRY || Opc == ARM::JUMPTABLE_ADDRS ||
801 Opc == ARM::JUMPTABLE_INSTS || Opc == ARM::JUMPTABLE_TBB ||
802 Opc == ARM::JUMPTABLE_TBH)
806 for (
unsigned op = 0, e =
I->getNumOperands();
op != e; ++
op)
807 if (
I->getOperand(
op).isCPI() ||
I->getOperand(
op).isJTI()) {
815 bool IsSoImm =
false;
823 case ARM::LEApcrelJT:
833 case ARM::t2LEApcrel:
834 case ARM::t2LEApcrelJT:
839 case ARM::tLEApcrelJT:
866 unsigned CPI =
I->getOperand(
op).getIndex();
867 if (
I->getOperand(
op).isJTI()) {
868 JumpTableUserIndices.insert(std::make_pair(CPI, CPUsers.size()));
869 CPI = JumpTableEntryIndices[CPI];
873 unsigned MaxOffs = ((1 <<
Bits)-1) * Scale;
874 CPUsers.push_back(CPUser(
I, CPEMI, MaxOffs, NegOk, IsSoImm));
877 CPEntry *CPE = findConstPoolEntry(CPI, CPEMI);
878 assert(CPE &&
"Cannot find a corresponding CPEntry!");
892 BasicBlockInfo &BBI = BBInfo[MBB->
getNumber()];
899 BBI.Size +=
TII->GetInstSizeInBytes(
I);
902 if (
I->isInlineAsm())
903 BBI.Unalign = isThumb ? 1 : 2;
905 else if (isThumb && mayOptimizeThumb2Instruction(
I))
919 unsigned ARMConstantIslands::getOffsetOf(
MachineInstr *
MI)
const {
925 unsigned Offset = BBInfo[MBB->
getNumber()].Offset;
929 assert(
I != MBB->
end() &&
"Didn't find MI in its own basic block?");
930 Offset +=
TII->GetInstSizeInBytes(
I);
951 BBInfo.insert(BBInfo.begin() + NewBB->
getNumber(), BasicBlockInfo());
956 std::lower_bound(WaterList.begin(), WaterList.end(), NewBB,
958 WaterList.insert(IP, NewBB);
981 unsigned Opc = isThumb ? (isThumb2 ? ARM::t2B : ARM::tB) : ARM::B;
998 MF->RenumberBlocks(NewBB);
1002 BBInfo.insert(BBInfo.begin() + NewBB->
getNumber(), BasicBlockInfo());
1009 std::lower_bound(WaterList.begin(), WaterList.end(), OrigBB,
1012 if (WaterBB == OrigBB)
1013 WaterList.
insert(std::next(IP), NewBB);
1015 WaterList.insert(IP, OrigBB);
1016 NewWaterList.insert(OrigBB);
1023 computeBlockSize(OrigBB);
1027 computeBlockSize(NewBB);
1030 adjustBBOffsetsAfter(OrigBB);
1038 unsigned ARMConstantIslands::getUserOffset(CPUser &U)
const {
1039 unsigned UserOffset = getOffsetOf(U.MI);
1040 const BasicBlockInfo &BBI = BBInfo[U.MI->getParent()->getNumber()];
1041 unsigned KnownBits = BBI.internalKnownBits();
1044 UserOffset += (isThumb ? 4 : 8);
1048 U.KnownAlignment = (KnownBits >= 2);
1053 if (isThumb && U.KnownAlignment)
1065 bool ARMConstantIslands::isOffsetInRange(
unsigned UserOffset,
1066 unsigned TrialOffset,
unsigned MaxDisp,
1067 bool NegativeOK,
bool IsSoImm) {
1068 if (UserOffset <= TrialOffset) {
1070 if (TrialOffset - UserOffset <= MaxDisp)
1073 }
else if (NegativeOK) {
1074 if (UserOffset - TrialOffset <= MaxDisp)
1085 bool ARMConstantIslands::isWaterInRange(
unsigned UserOffset,
1088 unsigned CPELogAlign = getCPELogAlign(U.CPEMI);
1089 unsigned CPEOffset = BBInfo[Water->
getNumber()].postOffset(CPELogAlign);
1090 unsigned NextBlockOffset, NextBlockAlignment;
1092 if (++NextBlock == MF->end()) {
1093 NextBlockOffset = BBInfo[Water->
getNumber()].postOffset();
1094 NextBlockAlignment = 0;
1096 NextBlockOffset = BBInfo[NextBlock->getNumber()].Offset;
1097 NextBlockAlignment = NextBlock->getAlignment();
1099 unsigned Size = U.CPEMI->getOperand(2).getImm();
1100 unsigned CPEEnd = CPEOffset + Size;
1105 if (CPEEnd > NextBlockOffset) {
1106 Growth = CPEEnd - NextBlockOffset;
1114 if (CPEOffset < UserOffset)
1115 UserOffset += Growth +
UnknownPadding(MF->getAlignment(), CPELogAlign);
1120 return isOffsetInRange(UserOffset, CPEOffset, U);
1125 bool ARMConstantIslands::isCPEntryInRange(
MachineInstr *MI,
unsigned UserOffset,
1127 bool NegOk,
bool DoDump) {
1128 unsigned CPEOffset = getOffsetOf(CPEMI);
1133 const BasicBlockInfo &BBI = BBInfo[Block];
1135 <<
" max delta=" << MaxDisp
1136 <<
format(
" insn address=%#x", UserOffset)
1137 <<
" in BB#" << Block <<
": "
1138 <<
format(
"%#x-%x\t", BBI.Offset, BBI.postOffset()) << *MI
1139 <<
format(
"CPE address=%#x offset=%+d: ", CPEOffset,
1140 int(CPEOffset-UserOffset));
1144 return isOffsetInRange(UserOffset, CPEOffset, MaxDisp, NegOk);
1166 for(
unsigned i = BBNum + 1, e = MF->getNumBlockIDs(); i < e; ++i) {
1169 unsigned LogAlign = MF->getBlockNumbered(i)->getAlignment();
1170 unsigned Offset = BBInfo[i - 1].postOffset(LogAlign);
1171 unsigned KnownBits = BBInfo[i - 1].postKnownBits(LogAlign);
1176 if (i > BBNum + 2 &&
1177 BBInfo[i].Offset == Offset &&
1178 BBInfo[i].KnownBits == KnownBits)
1181 BBInfo[i].Offset = Offset;
1182 BBInfo[i].KnownBits = KnownBits;
1191 bool ARMConstantIslands::decrementCPEReferenceCount(
unsigned CPI,
1194 CPEntry *CPE = findConstPoolEntry(CPI, CPEMI);
1195 assert(CPE &&
"Unexpected!");
1196 if (--CPE->RefCount == 0) {
1197 removeDeadCPEMI(CPEMI);
1198 CPE->CPEMI =
nullptr;
1205 unsigned ARMConstantIslands::getCombinedIndex(
const MachineInstr *CPEMI) {
1218 int ARMConstantIslands::findInRangeCPEntry(CPUser& U,
unsigned UserOffset)
1224 if (isCPEntryInRange(UserMI, UserOffset, CPEMI, U.getMaxDisp(), U.NegOk,
1231 unsigned CPI = getCombinedIndex(CPEMI);
1232 std::vector<CPEntry> &CPEs = CPEntries[CPI];
1233 for (
unsigned i = 0, e = CPEs.size(); i != e; ++i) {
1235 if (CPEs[i].CPEMI == CPEMI)
1238 if (CPEs[i].CPEMI ==
nullptr)
1240 if (isCPEntryInRange(UserMI, UserOffset, CPEs[i].CPEMI, U.getMaxDisp(),
1242 DEBUG(
dbgs() <<
"Replacing CPE#" << CPI <<
" with CPE#"
1243 << CPEs[i].CPI <<
"\n");
1245 U.CPEMI = CPEs[i].CPEMI;
1256 return decrementCPEReferenceCount(CPI, CPEMI) ? 2 : 1;
1267 return ((1<<10)-1)*2;
1269 return ((1<<23)-1)*2;
1274 return ((1<<23)-1)*4;
1285 bool ARMConstantIslands::findAvailableWater(CPUser &U,
unsigned UserOffset,
1286 water_iterator &WaterIter) {
1287 if (WaterList.empty())
1290 unsigned BestGrowth = ~0u;
1291 for (water_iterator IP = std::prev(WaterList.end()), B = WaterList.begin();;
1303 if (isWaterInRange(UserOffset, WaterBB, U, Growth) &&
1304 (WaterBB->
getNumber() < U.HighWaterMark->getNumber() ||
1305 NewWaterList.count(WaterBB) || WaterBB == U.MI->getParent()) &&
1306 Growth < BestGrowth) {
1308 BestGrowth = Growth;
1311 <<
" Growth=" << Growth <<
'\n');
1314 if (BestGrowth == 0)
1320 return BestGrowth != ~0u;
1330 void ARMConstantIslands::createNewWater(
unsigned CPUserIndex,
1331 unsigned UserOffset,
1333 CPUser &U = CPUsers[CPUserIndex];
1336 unsigned CPELogAlign = getCPELogAlign(CPEMI);
1338 const BasicBlockInfo &UserBBI = BBInfo[UserMBB->
getNumber()];
1346 unsigned Delta = isThumb1 ? 2 : 4;
1348 unsigned CPEOffset = UserBBI.postOffset(CPELogAlign) + Delta;
1350 if (isOffsetInRange(UserOffset, CPEOffset, U)) {
1352 <<
format(
", expected CPE offset %#x\n", CPEOffset));
1359 int UncondBr = isThumb ? ((isThumb2) ? ARM::t2B : ARM::tB) :
ARM::B;
1366 ImmBranches.push_back(ImmBranch(&UserMBB->
back(),
1367 MaxDisp,
false, UncondBr));
1368 computeBlockSize(UserMBB);
1369 adjustBBOffsetsAfter(UserMBB);
1389 unsigned LogAlign = MF->getAlignment();
1390 assert(LogAlign >= CPELogAlign &&
"Over-aligned constant pool entry");
1391 unsigned KnownBits = UserBBI.internalKnownBits();
1393 unsigned BaseInsertOffset = UserOffset + U.getMaxDisp() - UPad;
1400 BaseInsertOffset -= 4;
1403 <<
" la=" << LogAlign
1404 <<
" kb=" << KnownBits
1405 <<
" up=" << UPad <<
'\n');
1411 if (BaseInsertOffset + 8 >= UserBBI.postOffset()) {
1416 std::max(UserBBI.postOffset() - UPad - 8,
1417 UserOffset +
TII->GetInstSizeInBytes(UserMI) + 1);
1418 DEBUG(
dbgs() <<
format(
"Move inside block: %#x\n", BaseInsertOffset));
1420 unsigned EndInsertOffset = BaseInsertOffset + 4 + UPad +
1424 unsigned CPUIndex = CPUserIndex+1;
1425 unsigned NumCPUsers = CPUsers.size();
1427 for (
unsigned Offset = UserOffset+
TII->GetInstSizeInBytes(UserMI);
1428 Offset < BaseInsertOffset;
1429 Offset +=
TII->GetInstSizeInBytes(MI), MI = std::next(MI)) {
1430 assert(MI != UserMBB->
end() &&
"Fell off end of block");
1431 if (CPUIndex < NumCPUsers && CPUsers[CPUIndex].MI == MI) {
1432 CPUser &U = CPUsers[CPUIndex];
1433 if (!isOffsetInRange(Offset, EndInsertOffset, U)) {
1435 BaseInsertOffset -= 1u << LogAlign;
1436 EndInsertOffset -= 1u << LogAlign;
1442 EndInsertOffset += U.CPEMI->getOperand(2).getImm();
1447 if (MI->getOpcode() == ARM::t2IT)
1455 unsigned PredReg = 0;
1462 DEBUG(
unsigned PredReg;
1465 NewMBB = splitBlockBeforeInstr(MI);
1472 bool ARMConstantIslands::handleConstantPoolUser(
unsigned CPUserIndex) {
1473 CPUser &U = CPUsers[CPUserIndex];
1476 unsigned CPI = getCombinedIndex(CPEMI);
1479 unsigned UserOffset = getUserOffset(U);
1483 int result = findInRangeCPEntry(U, UserOffset);
1484 if (result==1)
return false;
1485 else if (result==2)
return true;
1489 unsigned ID = AFI->createPICLabelUId();
1495 if (findAvailableWater(U, UserOffset, IP)) {
1496 DEBUG(
dbgs() <<
"Found water in range\n");
1502 if (NewWaterList.erase(WaterBB))
1503 NewWaterList.
insert(NewIsland);
1511 createNewWater(CPUserIndex, UserOffset, NewMBB);
1519 IP = std::find(WaterList.begin(), WaterList.end(), WaterBB);
1520 if (IP != WaterList.end())
1521 NewWaterList.erase(WaterBB);
1524 NewWaterList.insert(NewIsland);
1531 if (IP != WaterList.end())
1532 WaterList.erase(IP);
1535 MF->insert(NewMBB, NewIsland);
1538 updateForInsertedWaterBlock(NewIsland);
1542 U.HighWaterMark = NewIsland;
1545 CPEntries[CPI].push_back(CPEntry(U.CPEMI, ID, 1));
1549 decrementCPEReferenceCount(CPI, CPEMI);
1555 BBInfo[NewIsland->
getNumber()].Size += Size;
1565 DEBUG(
dbgs() <<
" Moved CPE to #" << ID <<
" CPI=" << CPI
1573 void ARMConstantIslands::removeDeadCPEMI(
MachineInstr *CPEMI) {
1577 BBInfo[CPEBB->
getNumber()].Size -= Size;
1579 if (CPEBB->
empty()) {
1588 adjustBBOffsetsAfter(CPEBB);
1598 bool ARMConstantIslands::removeUnusedCPEntries() {
1599 unsigned MadeChange =
false;
1600 for (
unsigned i = 0, e = CPEntries.size(); i != e; ++i) {
1601 std::vector<CPEntry> &CPEs = CPEntries[i];
1602 for (
unsigned j = 0, ee = CPEs.size(); j != ee; ++j) {
1603 if (CPEs[j].RefCount == 0 && CPEs[j].CPEMI) {
1604 removeDeadCPEMI(CPEs[j].CPEMI);
1605 CPEs[j].CPEMI =
nullptr;
1617 unsigned PCAdj = isThumb ? 4 : 8;
1618 unsigned BrOffset = getOffsetOf(MI) + PCAdj;
1619 unsigned DestOffset = BBInfo[DestBB->
getNumber()].Offset;
1623 <<
" max delta=" << MaxDisp
1624 <<
" from " << getOffsetOf(MI) <<
" to " << DestOffset
1625 <<
" offset " <<
int(DestOffset-BrOffset) <<
"\t" << *
MI);
1627 if (BrOffset <= DestOffset) {
1629 if (DestOffset-BrOffset <= MaxDisp)
1632 if (BrOffset-DestOffset <= MaxDisp)
1640 bool ARMConstantIslands::fixupImmediateBr(ImmBranch &Br) {
1645 if (isBBInRange(MI, DestBB, Br.MaxDisp))
1649 return fixupUnconditionalBr(Br);
1650 return fixupConditionalBr(Br);
1658 ARMConstantIslands::fixupUnconditionalBr(ImmBranch &Br) {
1665 Br.MaxDisp = (1 << 21) * 2;
1668 adjustBBOffsetsAfter(MBB);
1672 DEBUG(
dbgs() <<
" Changed B to long jump " << *MI);
1681 ARMConstantIslands::fixupConditionalBr(ImmBranch &Br) {
1715 if (isBBInRange(MI, NewDest, Br.MaxDisp)) {
1716 DEBUG(
dbgs() <<
" Invert Bcc condition and swap its destination with "
1727 splitBlockBeforeInstr(MI);
1730 int delta =
TII->GetInstSizeInBytes(&MBB->
back());
1738 <<
" also invert condition and change dest. to BB#"
1745 Br.MI = &MBB->
back();
1754 ImmBranches.push_back(ImmBranch(&MBB->
back(), MaxDisp,
false, Br.UncondBr));
1759 adjustBBOffsetsAfter(MBB);
1766 bool ARMConstantIslands::undoLRSpillRestore() {
1767 bool MadeChange =
false;
1768 for (
unsigned i = 0, e = PushPopMIs.size(); i != e; ++i) {
1788 ARMConstantIslands::mayOptimizeThumb2Instruction(
const MachineInstr *MI)
const {
1791 case ARM::t2LEApcrel:
1804 bool ARMConstantIslands::optimizeThumb2Instructions() {
1805 bool MadeChange =
false;
1808 for (
unsigned i = 0, e = CPUsers.size(); i != e; ++i) {
1809 CPUser &U = CPUsers[i];
1810 unsigned Opcode = U.MI->getOpcode();
1811 unsigned NewOpc = 0;
1816 case ARM::t2LEApcrel:
1818 NewOpc = ARM::tLEApcrel;
1825 NewOpc = ARM::tLDRpci;
1835 unsigned UserOffset = getUserOffset(U);
1836 unsigned MaxOffs = ((1 <<
Bits) - 1) * Scale;
1839 if (!U.KnownAlignment)
1843 if (isCPEntryInRange(U.MI, UserOffset, U.CPEMI, MaxOffs,
false,
true)) {
1845 U.MI->setDesc(
TII->get(NewOpc));
1848 adjustBBOffsetsAfter(MBB);
1854 MadeChange |= optimizeThumb2Branches();
1855 MadeChange |= optimizeThumb2JumpTables();
1859 bool ARMConstantIslands::optimizeThumb2Branches() {
1860 bool MadeChange =
false;
1867 for (
unsigned i = ImmBranches.size(); i != 0; --i) {
1868 ImmBranch &Br = ImmBranches[i-1];
1869 unsigned Opcode = Br.MI->getOpcode();
1870 unsigned NewOpc = 0;
1888 unsigned MaxOffs = ((1 << (Bits-1))-1) * Scale;
1890 if (isBBInRange(Br.MI, DestBB, MaxOffs)) {
1891 DEBUG(
dbgs() <<
"Shrink branch: " << *Br.MI);
1892 Br.MI->setDesc(
TII->get(NewOpc));
1895 adjustBBOffsetsAfter(MBB);
1901 Opcode = Br.MI->getOpcode();
1902 if (Opcode != ARM::tBcc)
1907 if (!Br.MI->killsRegister(ARM::CPSR))
1911 unsigned PredReg = 0;
1916 NewOpc = ARM::tCBNZ;
1922 unsigned BrOffset = getOffsetOf(Br.MI) + 4 - 2;
1923 unsigned DestOffset = BBInfo[DestBB->
getNumber()].Offset;
1924 if (BrOffset < DestOffset && (DestOffset - BrOffset) <= 126) {
1926 if (CmpMI != Br.MI->getParent()->begin()) {
1928 if (CmpMI->getOpcode() == ARM::tCMPi8) {
1929 unsigned Reg = CmpMI->getOperand(0).getReg();
1932 CmpMI->getOperand(1).getImm() == 0 &&
1935 DEBUG(
dbgs() <<
"Fold: " << *CmpMI <<
" and: " << *Br.MI);
1937 BuildMI(*MBB, CmpMI, Br.MI->getDebugLoc(),
TII->get(NewOpc))
1938 .addReg(Reg).
addMBB(DestBB,Br.MI->getOperand(0).getTargetFlags());
1940 Br.MI->eraseFromParent();
1943 adjustBBOffsetsAfter(MBB);
1976 bool ARMConstantIslands::preserveBaseRegister(
MachineInstr *JumpMI,
1980 bool &BaseRegKill) {
2003 CanDeleteLEA =
true;
2004 BaseRegKill =
false;
2007 for (++
I; &*
I != JumpMI; ++
I) {
2013 for (
unsigned K = 0, E =
I->getNumOperands(); K != E; ++K) {
2020 BaseRegKill = BaseRegKill || MO.
isKill();
2021 CanDeleteLEA =
false;
2031 for (++
I; &*
I != JumpMI; ++
I) {
2032 for (
unsigned K = 0, E =
I->getNumOperands(); K != E; ++K) {
2039 RemovableAdd =
nullptr;
2046 }
else if (BaseReg == EntryReg) {
2067 return MBB != MF->
end() && MBB->begin() != MBB->end() &&
2068 &*MBB->begin() == CPEMI;
2073 bool ARMConstantIslands::optimizeThumb2JumpTables() {
2074 bool MadeChange =
false;
2079 if (!MJTI)
return false;
2081 const std::vector<MachineJumpTableEntry> &JT = MJTI->
getJumpTables();
2082 for (
unsigned i = 0, e = T2JumpTables.size(); i != e; ++i) {
2086 unsigned JTOpIdx = NumOps - (MI->
isPredicable() ? 2 : 1);
2088 unsigned JTI = JTOP.getIndex();
2089 assert(JTI < JT.size());
2092 bool HalfWordOk =
true;
2093 unsigned JTOffset = getOffsetOf(MI) + 4;
2094 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
2095 for (
unsigned j = 0, ee = JTBBs.size(); j != ee; ++j) {
2097 unsigned DstOffset = BBInfo[MBB->
getNumber()].Offset;
2100 if (ByteOk && (DstOffset - JTOffset) > ((1<<8)-1)*2)
2102 unsigned TBHLimit = ((1<<16)-1)*2;
2103 if (HalfWordOk && (DstOffset - JTOffset) > TBHLimit)
2105 if (!ByteOk && !HalfWordOk)
2109 if (!ByteOk && !HalfWordOk)
2118 CPUser &
User = CPUsers[JumpTableUserIndices[JTI]];
2119 unsigned DeadSize = 0;
2120 bool CanDeleteLEA =
false;
2121 bool BaseRegKill =
false;
2122 bool PreservedBaseReg =
2123 preserveBaseRegister(MI, User.MI, DeadSize, CanDeleteLEA, BaseRegKill);
2130 unsigned Opc = ByteOk ? ARM::t2TBB_JT : ARM::t2TBH_JT;
2134 .addReg(User.MI->getOperand(0).getReg(),
2137 .addJumpTableIndex(JTI, JTOP.getTargetFlags())
2141 unsigned JTOpc = ByteOk ? ARM::JUMPTABLE_TBB : ARM::JUMPTABLE_TBH;
2149 User.MI->eraseFromParent();
2157 User.IsSoImm =
false;
2158 User.KnownAlignment =
false;
2162 int CPEntryIdx = JumpTableEntryIndices[JTI];
2163 auto &CPEs = CPEntries[CPEntryIdx];
2164 auto Entry = std::find_if(CPEs.begin(), CPEs.end(), [&](CPEntry &E) {
2165 return E.CPEMI == User.CPEMI;
2168 CPUsers.emplace_back(CPUser(NewJTMI, User.CPEMI, 4,
false,
false));
2172 unsigned NewSize =
TII->GetInstSizeInBytes(NewJTMI);
2173 unsigned OrigSize =
TII->GetInstSizeInBytes(MI);
2176 int Delta = OrigSize - NewSize + DeadSize;
2178 adjustBBOffsetsAfter(MBB);
2189 bool ARMConstantIslands::reorderThumb2JumpTables() {
2190 bool MadeChange =
false;
2193 if (!MJTI)
return false;
2195 const std::vector<MachineJumpTableEntry> &JT = MJTI->
getJumpTables();
2196 for (
unsigned i = 0, e = T2JumpTables.size(); i != e; ++i) {
2200 unsigned JTOpIdx = NumOps - (MI->
isPredicable() ? 2 : 1);
2202 unsigned JTI = JTOP.getIndex();
2203 assert(JTI < JT.size());
2209 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
2210 for (
unsigned j = 0, ee = JTBBs.size(); j != ee; ++j) {
2214 if (DTNumber < JTNumber) {
2218 adjustJTTargetBlockForward(MBB, MI->
getParent());
2247 if (!B && Cond.
empty() && BB != MF->
begin() &&
2250 OldPrior->updateTerminator();
2253 MF->RenumberBlocks();
2267 assert (isThumb2 &&
"Adjusting for TB[BH] but not in Thumb2?");
2272 MF->RenumberBlocks(NewBB);
unsigned succ_size() const
const MachineFunction * getParent() const
getParent - Return the MachineFunction containing this basic block.
A parsed version of the target data layout string in and methods for querying it. ...
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
bool AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
STATISTIC(NumFunctions,"Total number of functions")
void RenumberBlocks(MachineBasicBlock *MBBFrom=nullptr)
RenumberBlocks - This discards all of the MachineBasicBlock numbers and recomputes them...
MachineBasicBlock * getMBB() const
int getNumber() const
getNumber - MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a M...
Describe properties that are true of each instruction in the target description file.
bool isPredicable(QueryType Type=AllInBundle) const
Return true if this instruction has a predicate operand that controls execution.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
void moveAfter(MachineBasicBlock *NewBefore)
FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > Tys=None)
Return the function type for an intrinsic.
static bool isThumb(const MCSubtargetInfo &STI)
void setAlignment(unsigned Align)
setAlignment - Set alignment of the basic block.
bool ReplaceMBBInJumpTable(unsigned Idx, MachineBasicBlock *Old, MachineBasicBlock *New)
ReplaceMBBInJumpTable - If Old is a target of the jump tables, update the jump table to branch to New...
ARMCC::CondCodes getITInstrPredicate(const MachineInstr *MI, unsigned &PredReg)
getITInstrPredicate - Valid only in Thumb2 mode.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, bool gen_crash_diag=true)
Reports a serious error, calling any installed error handler.
bool isLRSpilledForFarJump() const
static bool CompareMBBNumbers(const MachineBasicBlock *LHS, const MachineBasicBlock *RHS)
CompareMBBNumbers - Little predicate function to sort the WaterList by MBB ID.
static bool isSimpleIndexCalc(MachineInstr &I, unsigned EntryReg, unsigned BaseReg)
const std::vector< MachineJumpTableEntry > & getJumpTables() const
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Reg
All possible values of the reg field in the ModR/M byte.
const MachineInstrBuilder & addImm(int64_t Val) const
addImm - Add a new immediate operand.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
bool isCPI() const
isCPI - Tests if this is a MO_ConstantPoolIndex operand.
FunctionPass * createARMConstantIslandPass()
createARMConstantIslandPass - returns an instance of the constpool island pass.
void transferSuccessors(MachineBasicBlock *fromMBB)
transferSuccessors - Transfers all the successors from MBB to this machine basic block (i...
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
iterator getLastNonDebugInstr()
getLastNonDebugInstr - returns an iterator to the last non-debug instruction in the basic block...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
unsigned getKillRegState(bool B)
const BasicBlock * getBasicBlock() const
getBasicBlock - Return the LLVM basic block that this instance corresponded to originally.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
format_object< Ts...> format(const char *Fmt, const Ts &...Vals)
These are helper functions used to produce formatted output.
bundle_iterator< MachineInstr, instr_iterator > iterator
initializer< Ty > init(const Ty &Val)
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
ARMCC::CondCodes getInstrPredicate(const MachineInstr *MI, unsigned &PredReg)
getInstrPredicate - If instruction is predicated, returns its predicate condition, otherwise returns AL.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
const MachineOperand & getOperand(unsigned i) const
void ensureAlignment(unsigned A)
ensureAlignment - Make sure the function is at least 1 << A bytes aligned.
void setMBB(MachineBasicBlock *MBB)
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
static bool BBIsJumpedOver(MachineBasicBlock *MBB)
BBIsJumpedOver - Return true of the specified basic block's only predecessor unconditionally branches...
void setImm(int64_t immVal)
FunctionPass class - This class is used to implement most global optimizations.
static unsigned UnknownPadding(unsigned LogAlign, unsigned KnownBits)
UnknownPadding - Return the worst case padding that could result from unknown offset bits...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID)
BuildMI - Builder interface.
succ_iterator succ_begin()
void removeSuccessor(MachineBasicBlock *succ)
removeSuccessor - Remove successor from the successors list of this MachineBasicBlock.
pred_iterator pred_begin()
void setIsKill(bool Val=true)
void setDesc(const MCInstrDesc &tid)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one...
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned char TargetFlags=0) const
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
void updateTerminator()
updateTerminator - Update the terminator instructions in block to account for changes to the layout...
static cl::opt< bool > AdjustJumpTableBlocks("arm-adjust-jump-tables", cl::Hidden, cl::init(true), cl::desc("Adjust basic block layout to better use TB[BH]"))
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
static bool isARMLowRegister(unsigned Reg)
isARMLowRegister - Returns true if the register is a low register (r0-r7).
Representation of each machine instruction.
static CondCodes getOppositeCondition(CondCodes CC)
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
void setReg(unsigned Reg)
Change the register this operand corresponds to.
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned char TargetFlags=0) const
void push_back(MachineInstr *MI)
static unsigned getUnconditionalBrDisp(int Opc)
getUnconditionalBrDisp - Returns the maximum displacement that can fit in the specific unconditional ...
unsigned getReg() const
getReg - Returns the register number.
static bool jumpTableFollowsTB(MachineInstr *JTMI, MachineInstr *CPEMI)
Returns whether CPEMI is the first instruction in the block immediately following JTMI (assumed to be...
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
uint64_t OffsetToAlignment(uint64_t Value, uint64_t Align)
Returns the offset to the next integer (mod 2**64) that is greater than or equal to Value and is a mu...
const MachineInstrBuilder & addOperand(const MachineOperand &MO) const
BasicBlockListType::iterator iterator
bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
static bool BBHasFallthrough(MachineBasicBlock *MBB)
BBHasFallthrough - Return true if the specified basic block can fallthrough into the block immediatel...
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
addReg - Add a new virtual register operand...
void addSuccessor(MachineBasicBlock *succ, uint32_t weight=0)
addSuccessor - Add succ as a successor of this MachineBasicBlock.
unsigned pred_size() const
unsigned getAlignment() const
getAlignment - Return alignment of the basic block.