26 #define DEBUG_TYPE "hexagon-isel"
31 cl::desc(
"Rebalance address calculation trees to improve "
32 "instruction selection"));
39 cl::desc(
"Rebalance address tree only if this allows optimizations"));
44 cl::init(
false),
cl::desc(
"Rebalance address tree only if it is imbalanced"));
74 void PreprocessISelDAG()
override;
75 void EmitFunctionEntryCode()
override;
86 return "Hexagon DAG->DAG Pattern Instruction Selection";
97 void SelectFrameIndex(
SDNode *
N);
100 bool SelectInlineAsmMemoryOperand(
const SDValue &
Op,
101 unsigned ConstraintID,
102 std::vector<SDValue> &OutOps)
override;
110 void SelectZeroExtend(
SDNode *
N);
111 void SelectIntrinsicWChain(
SDNode *
N);
112 void SelectIntrinsicWOChain(
SDNode *
N);
113 void SelectConstant(
SDNode *
N);
114 void SelectConstantFP(
SDNode *
N);
118 #include "HexagonGenDAGISel.inc"
121 bool isValueExtension(
const SDValue &Val,
unsigned FromBits,
SDValue &Src);
122 bool isOrEquivalentToAdd(
const SDNode *
N)
const;
123 bool isAlignedMemNode(
const MemSDNode *
N)
const;
124 bool isPositiveHalfWord(
const SDNode *
N)
const;
133 unsigned getUsesInFunction(
const Value *V);
135 void rebalanceAddressTrees();
146 return new HexagonDAGToDAGISel(TM, OptLevel);
155 case Intrinsic::hexagon_C2_cmpeq:
156 case Intrinsic::hexagon_C2_cmpgt:
157 case Intrinsic::hexagon_C2_cmpgtu:
158 case Intrinsic::hexagon_C2_cmpgtup:
159 case Intrinsic::hexagon_C2_cmpgtp:
160 case Intrinsic::hexagon_C2_cmpeqp:
161 case Intrinsic::hexagon_C2_bitsset:
162 case Intrinsic::hexagon_C2_bitsclr:
163 case Intrinsic::hexagon_C2_cmpeqi:
164 case Intrinsic::hexagon_C2_cmpgti:
165 case Intrinsic::hexagon_C2_cmpgtui:
166 case Intrinsic::hexagon_C2_cmpgei:
167 case Intrinsic::hexagon_C2_cmpgeui:
168 case Intrinsic::hexagon_C2_cmplt:
169 case Intrinsic::hexagon_C2_cmpltu:
170 case Intrinsic::hexagon_C2_bitsclri:
171 case Intrinsic::hexagon_C2_and:
172 case Intrinsic::hexagon_C2_or:
173 case Intrinsic::hexagon_C2_xor:
174 case Intrinsic::hexagon_C2_andn:
175 case Intrinsic::hexagon_C2_not:
176 case Intrinsic::hexagon_C2_orn:
177 case Intrinsic::hexagon_C2_pxfer_map:
178 case Intrinsic::hexagon_C2_any8:
179 case Intrinsic::hexagon_C2_all8:
180 case Intrinsic::hexagon_A2_vcmpbeq:
181 case Intrinsic::hexagon_A2_vcmpbgtu:
182 case Intrinsic::hexagon_A2_vcmpheq:
183 case Intrinsic::hexagon_A2_vcmphgt:
184 case Intrinsic::hexagon_A2_vcmphgtu:
185 case Intrinsic::hexagon_A2_vcmpweq:
186 case Intrinsic::hexagon_A2_vcmpwgt:
187 case Intrinsic::hexagon_A2_vcmpwgtu:
188 case Intrinsic::hexagon_C2_tfrrp:
189 case Intrinsic::hexagon_S2_tstbit_i:
190 case Intrinsic::hexagon_S2_tstbit_r:
199 int32_t Inc = cast<ConstantSDNode>(Offset.
getNode())->getSExtValue();
207 bool IsValidInc = HII->isValidAutoIncImm(LoadedVT, Inc);
213 Opcode = IsValidInc ? Hexagon::L2_loadrub_pi : Hexagon::L2_loadrub_io;
215 Opcode = IsValidInc ? Hexagon::L2_loadrb_pi : Hexagon::L2_loadrb_io;
219 Opcode = IsValidInc ? Hexagon::L2_loadruh_pi : Hexagon::L2_loadruh_io;
221 Opcode = IsValidInc ? Hexagon::L2_loadrh_pi : Hexagon::L2_loadrh_io;
224 Opcode = IsValidInc ? Hexagon::L2_loadri_pi : Hexagon::L2_loadri_io;
227 Opcode = IsValidInc ? Hexagon::L2_loadrd_pi : Hexagon::L2_loadrd_io;
234 if (isAlignedMemNode(LD))
235 Opcode = IsValidInc ? Hexagon::V6_vL32b_pi : Hexagon::V6_vL32b_ai;
237 Opcode = IsValidInc ? Hexagon::V6_vL32Ub_pi : Hexagon::V6_vL32Ub_ai;
244 if (isAlignedMemNode(LD))
245 Opcode = IsValidInc ? Hexagon::V6_vL32b_pi_128B
246 : Hexagon::V6_vL32b_ai_128B;
248 Opcode = IsValidInc ? Hexagon::V6_vL32Ub_pi_128B
249 : Hexagon::V6_vL32Ub_ai_128B;
263 return CurDAG->getMachineNode(Hexagon::A4_combineir, dl,
MVT::i64,
267 return CurDAG->getMachineNode(Hexagon::A2_sxtw, dl,
MVT::i64,
309 ReplaceUses(From, To, 3);
310 CurDAG->RemoveDeadNode(LD);
319 unsigned IntNo = cast<ConstantSDNode>(IntN->
getOperand(1))->getZExtValue();
321 static std::map<unsigned,unsigned> LoadPciMap = {
322 { Intrinsic::hexagon_circ_ldb, Hexagon::L2_loadrb_pci },
323 { Intrinsic::hexagon_circ_ldub, Hexagon::L2_loadrub_pci },
324 { Intrinsic::hexagon_circ_ldh, Hexagon::L2_loadrh_pci },
325 { Intrinsic::hexagon_circ_lduh, Hexagon::L2_loadruh_pci },
326 { Intrinsic::hexagon_circ_ldw, Hexagon::L2_loadri_pci },
327 { Intrinsic::hexagon_circ_ldd, Hexagon::L2_loadrd_pci },
329 auto FLC = LoadPciMap.find(IntNo);
330 if (FLC != LoadPciMap.end()) {
331 SDNode *Mod = CurDAG->getMachineNode(Hexagon::A2_tfrrcr, dl,
MVT::i32,
336 auto Inc = cast<ConstantSDNode>(IntN->
getOperand(5));
338 MachineSDNode *Res = CurDAG->getMachineNode(FLC->second, dl, RTys,
343 static std::map<unsigned,unsigned> LoadPbrMap = {
344 { Intrinsic::hexagon_brev_ldb, Hexagon::L2_loadrb_pbr },
345 { Intrinsic::hexagon_brev_ldub, Hexagon::L2_loadrub_pbr },
346 { Intrinsic::hexagon_brev_ldh, Hexagon::L2_loadrh_pbr },
347 { Intrinsic::hexagon_brev_lduh, Hexagon::L2_loadruh_pbr },
348 { Intrinsic::hexagon_brev_ldw, Hexagon::L2_loadri_pbr },
349 { Intrinsic::hexagon_brev_ldd, Hexagon::L2_loadrd_pbr },
351 auto FLB = LoadPbrMap.find(IntNo);
352 if (FLB != LoadPbrMap.end()) {
353 SDNode *Mod = CurDAG->getMachineNode(Hexagon::A2_tfrrcr, dl,
MVT::i32,
358 MachineSDNode *Res = CurDAG->getMachineNode(FLB->second, dl, RTys,
374 unsigned Size = 1U << (SizeBits-1);
382 TS = CurDAG->getStore(
SDValue(LoadN, 2), dl,
SDValue(LoadN, 0), Loc, PI,
385 TS = CurDAG->getTruncStore(
SDValue(LoadN, 2), dl,
SDValue(LoadN, 0), Loc,
386 PI, MVT::getIntegerVT(Size * 8), Size);
392 StoreN = Handle.getValue().getNode();
401 bool HexagonDAGToDAGISel::tryLoadOfLoadIntrinsic(
LoadSDNode *N) {
432 switch (cast<ConstantSDNode>(C->
getOperand(1))->getZExtValue()) {
433 case Intrinsic::hexagon_brev_ldub:
434 case Intrinsic::hexagon_brev_lduh:
435 case Intrinsic::hexagon_circ_ldub:
436 case Intrinsic::hexagon_circ_lduh:
439 case Intrinsic::hexagon_brev_ldw:
440 case Intrinsic::hexagon_brev_ldd:
441 case Intrinsic::hexagon_circ_ldw:
442 case Intrinsic::hexagon_circ_ldd:
458 SDNode *S = StoreInstrForLoadIntrinsic(L, C);
465 CurDAG->RemoveDeadNode(C);
472 void HexagonDAGToDAGISel::SelectLoad(
SDNode *N) {
479 SelectIndexedLoad(LD, dl);
484 if (tryLoadOfLoadIntrinsic(LD))
496 int32_t Inc = cast<ConstantSDNode>(Offset.
getNode())->getSExtValue();
500 bool IsValidInc = HII->isValidAutoIncImm(StoredVT, Inc);
506 Opcode = IsValidInc ? Hexagon::S2_storerb_pi : Hexagon::S2_storerb_io;
509 Opcode = IsValidInc ? Hexagon::S2_storerh_pi : Hexagon::S2_storerh_io;
512 Opcode = IsValidInc ? Hexagon::S2_storeri_pi : Hexagon::S2_storeri_io;
515 Opcode = IsValidInc ? Hexagon::S2_storerd_pi : Hexagon::S2_storerd_io;
522 if (isAlignedMemNode(ST))
523 Opcode = IsValidInc ? Hexagon::V6_vS32b_pi : Hexagon::V6_vS32b_ai;
525 Opcode = IsValidInc ? Hexagon::V6_vS32Ub_pi : Hexagon::V6_vS32Ub_ai;
532 if (isAlignedMemNode(ST))
533 Opcode = IsValidInc ? Hexagon::V6_vS32b_pi_128B
534 : Hexagon::V6_vS32b_ai_128B;
536 Opcode = IsValidInc ? Hexagon::V6_vS32Ub_pi_128B
537 : Hexagon::V6_vS32Ub_ai_128B;
545 Value = CurDAG->getTargetExtractSubreg(Hexagon::isub_lo,
546 dl, MVT::i32, Value);
549 SDValue IncV = CurDAG->getTargetConstant(Inc, dl, MVT::i32);
559 SDValue Ops[] = { Base, IncV, Value, Chain };
566 SDValue Zero = CurDAG->getTargetConstant(0, dl, MVT::i32);
571 MachineSDNode *A = CurDAG->getMachineNode(Hexagon::A2_addi, dl, MVT::i32,
576 ReplaceUses(From, To, 2);
577 CurDAG->RemoveDeadNode(ST);
580 void HexagonDAGToDAGISel::SelectStore(
SDNode *N) {
587 SelectIndexedStore(ST, dl);
594 void HexagonDAGToDAGISel::SelectMul(
SDNode *N) {
631 SDValue TargetConst0 = CurDAG->getTargetConstant(0, dl, MVT::i32);
632 OP0 =
SDValue(CurDAG->getMachineNode(Hexagon::L2_loadri_io, dl, MVT::i32,
658 SDValue TargetConst0 = CurDAG->getTargetConstant(0, dl, MVT::i32);
659 OP1 =
SDValue(CurDAG->getMachineNode(Hexagon::L2_loadri_io, dl, MVT::i32,
669 SDNode *Result = CurDAG->getMachineNode(Hexagon::M2_dpmpyss_s0, dl,
671 ReplaceNode(N, Result);
678 void HexagonDAGToDAGISel::SelectSHL(
SDNode *N) {
683 auto Default = [
this,
N] () ->
void { SelectCode(N); };
689 int32_t ShlConst = cast<ConstantSDNode>(Shl_1)->getSExtValue();
696 int32_t ValConst = C->getSExtValue() << ShlConst;
697 if (isInt<9>(ValConst)) {
698 SDValue Val = CurDAG->getTargetConstant(ValConst, dl, MVT::i32);
699 SDNode *Result = CurDAG->getMachineNode(Hexagon::M2_mpysmi, dl,
700 MVT::i32, Mul_0, Val);
701 ReplaceNode(N, Result);
717 int32_t ValConst = 1 << (ShlConst + C2->getSExtValue());
718 if (isInt<9>(-ValConst)) {
719 SDValue Val = CurDAG->getTargetConstant(-ValConst, dl, MVT::i32);
720 SDNode *Result = CurDAG->getMachineNode(Hexagon::M2_mpysmi, dl,
721 MVT::i32, Shl2_0, Val);
722 ReplaceNode(N, Result);
743 void HexagonDAGToDAGISel::SelectZeroExtend(
SDNode *N) {
752 SDNode *
Mask = CurDAG->getMachineNode(Hexagon::C2_mask, dl, MVT::i64, Op0);
756 uint64_t MV = 0,
Bit = 1;
757 for (
unsigned i = 0;
i <
NE; ++
i) {
761 SDValue Ones = CurDAG->getTargetConstant(MV, dl, MVT::i64);
762 SDNode *OnesReg = CurDAG->getMachineNode(Hexagon::CONST64, dl,
765 SDNode *
And = CurDAG->getMachineNode(Hexagon::A2_andp, dl, MVT::i64,
767 SDValue SubR = CurDAG->getTargetConstant(Hexagon::isub_lo, dl, MVT::i32);
768 ReplaceNode(N, CurDAG->getMachineNode(Hexagon::EXTRACT_SUBREG, dl, ExVT,
773 CurDAG->getMachineNode(Hexagon::A2_andp, dl, ExVT,
780 unsigned ID = cast<ConstantSDNode>(Int->
getOperand(0))->getZExtValue();
785 SDValue TargetConst0 = CurDAG->getTargetConstant(0, dl, MVT::i32);
786 SDNode *Result_1 = CurDAG->getMachineNode(Hexagon::C2_tfrpr, dl,
788 SDNode *Result_2 = CurDAG->getMachineNode(Hexagon::A2_tfrsi, dl,
789 MVT::i32, TargetConst0);
790 SDNode *Result_3 = CurDAG->getMachineNode(Hexagon::A2_combinew, dl,
794 ReplaceNode(N, Result_3);
799 SDNode* RsPd = CurDAG->getMachineNode(Hexagon::C2_tfrpr, dl,
801 ReplaceNode(N, RsPd);
814 void HexagonDAGToDAGISel::SelectIntrinsicWChain(
SDNode *N) {
816 StoreInstrForLoadIntrinsic(L, N);
817 CurDAG->RemoveDeadNode(N);
823 void HexagonDAGToDAGISel::SelectIntrinsicWOChain(
SDNode *N) {
824 unsigned IID = cast<ConstantSDNode>(N->
getOperand(0))->getZExtValue();
827 case Intrinsic::hexagon_S2_vsplatrb:
830 case Intrinsic::hexagon_S2_vsplatrh:
840 if (isValueExtension(V, Bits, U)) {
853 void HexagonDAGToDAGISel::SelectConstantFP(
SDNode *N) {
858 SDValue V = CurDAG->getTargetConstant(A.getZExtValue(), dl, MVT::i32);
859 ReplaceNode(N, CurDAG->getMachineNode(Hexagon::A2_tfrsi, dl, MVT::f32, V));
863 SDValue V = CurDAG->getTargetConstant(A.getZExtValue(), dl, MVT::i64);
864 ReplaceNode(N, CurDAG->getMachineNode(Hexagon::CONST64, dl, MVT::f64, V));
874 void HexagonDAGToDAGISel::SelectConstant(
SDNode *N) {
876 assert(!(cast<ConstantSDNode>(N)->getZExtValue() >> 1));
877 unsigned Opc = (cast<ConstantSDNode>(
N)->getSExtValue() != 0)
880 ReplaceNode(N, CurDAG->getMachineNode(Opc,
SDLoc(N), MVT::i1));
888 void HexagonDAGToDAGISel::SelectFrameIndex(
SDNode *N) {
891 int FX = cast<FrameIndexSDNode>(
N)->getIndex();
894 SDValue FI = CurDAG->getTargetFrameIndex(FX, MVT::i32);
896 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
905 R = CurDAG->getMachineNode(Hexagon::PS_fi, DL, MVT::i32, FI, Zero);
910 SDValue Ops[] = { CurDAG->getCopyFromReg(CH, DL, AR, MVT::i32), FI, Zero };
911 R = CurDAG->getMachineNode(Hexagon::PS_fia, DL, MVT::i32, Ops);
918 void HexagonDAGToDAGISel::SelectBitcast(
SDNode *N) {
930 CurDAG->RemoveDeadNode(N);
978 SelectIntrinsicWChain(N);
982 SelectIntrinsicWOChain(N);
989 bool HexagonDAGToDAGISel::
990 SelectInlineAsmMemoryOperand(
const SDValue &
Op,
unsigned ConstraintID,
991 std::vector<SDValue> &OutOps) {
994 switch (ConstraintID) {
997 case InlineAsm::Constraint_i:
998 case InlineAsm::Constraint_o:
999 case InlineAsm::Constraint_v:
1000 case InlineAsm::Constraint_m:
1001 if (SelectAddrFI(Inp, Res))
1002 OutOps.push_back(Res);
1004 OutOps.push_back(Inp);
1008 OutOps.push_back(CurDAG->getTargetConstant(0,
SDLoc(Op), MVT::i32));
1013 void HexagonDAGToDAGISel::PreprocessISelDAG() {
1015 std::vector<SDNode*> Nodes;
1017 Nodes.push_back(&Node);
1022 for (
auto I : Nodes) {
1023 if (I->getOpcode() !=
ISD::OR)
1026 auto IsZero = [] (
const SDValue &V) ->
bool {
1028 return SC->isNullValue();
1031 auto IsSelect0 = [IsZero] (
const SDValue &
Op) ->
bool {
1038 EVT VT = I->getValueType(0);
1039 bool SelN0 = IsSelect0(N0);
1040 SDValue SOp = SelN0 ? N0 : N1;
1041 SDValue VOp = SelN0 ? N1 : N0;
1052 }
else if (IsZero(SX)) {
1066 for (
auto I : Nodes) {
1098 if (EV % (1 << CV) != 0)
1100 unsigned DV = EV / (1 << CV);
1114 rebalanceAddressTrees();
1117 dbgs() <<
"************* SelectionDAG after preprocessing: ***********\n";
1119 dbgs() <<
"************* End SelectionDAG after preprocessing ********\n";
1124 void HexagonDAGToDAGISel::EmitFunctionEntryCode() {
1126 auto &HFI = *HST.getFrameLowering();
1132 unsigned AR = FuncInfo->CreateReg(MVT::i32);
1143 auto &HFI = *HST->getFrameLowering();
1145 int FX = cast<FrameIndexSDNode>(
N)->getIndex();
1148 R = CurDAG->getTargetFrameIndex(FX, MVT::i32);
1152 inline bool HexagonDAGToDAGISel::SelectAddrGA(
SDValue &N,
SDValue &R) {
1153 return SelectGlobalAddress(N, R,
false);
1156 inline bool HexagonDAGToDAGISel::SelectAddrGP(
SDValue &N,
SDValue &R) {
1157 return SelectGlobalAddress(N, R,
true);
1160 bool HexagonDAGToDAGISel::SelectGlobalAddress(
SDValue &N,
SDValue &R,
1175 uint64_t NewOff = GA->getOffset() + (uint64_t)
Const->getSExtValue();
1176 R = CurDAG->getTargetGlobalAddress(GA->getGlobal(),
SDLoc(Const),
1201 bool HexagonDAGToDAGISel::isValueExtension(
const SDValue &Val,
1202 unsigned FromBits,
SDValue &Src) {
1229 uint64_t FromMask = (1 << FromBits) - 1;
1231 if (C->getZExtValue() == FromMask) {
1237 if (C->getZExtValue() == FromMask) {
1247 uint64_t FromMask = (1 << FromBits) - 1;
1249 if ((C->getZExtValue() & FromMask) == 0) {
1255 if ((C->getZExtValue() & FromMask) == 0) {
1268 bool HexagonDAGToDAGISel::isOrEquivalentToAdd(
const SDNode *N)
const {
1274 if (
auto *FN = dyn_cast<FrameIndexSDNode>(N->
getOperand(0))) {
1278 int32_t Off = C->getSExtValue();
1281 return (Off >= 0) && (((A-1) & Off) ==
unsigned(Off));
1286 bool HexagonDAGToDAGISel::isAlignedMemNode(
const MemSDNode *N)
const {
1291 bool HexagonDAGToDAGISel::isPositiveHalfWord(
const SDNode *N)
const {
1292 if (
const ConstantSDNode *CN = dyn_cast<const ConstantSDNode>(N)) {
1321 int HexagonDAGToDAGISel::getWeight(
SDNode *N) {
1324 assert(RootWeights.count(N) &&
"Cannot get weight of unseen root!");
1325 assert(RootWeights[N] != -1 &&
"Cannot get weight of unvisited root!");
1326 assert(RootWeights[N] != -2 &&
"Cannot get weight of RAWU'd root!");
1327 return RootWeights[
N];
1330 int HexagonDAGToDAGISel::getHeight(
SDNode *N) {
1333 assert(RootWeights.count(N) && RootWeights[
N] >= 0 &&
1334 "Cannot query height of unvisited/RAUW'd node!");
1335 return RootHeights[
N];
1339 struct WeightedLeaf {
1344 WeightedLeaf() : Value(
SDValue()) { }
1346 WeightedLeaf(
SDValue Value,
int Weight,
int InsertionOrder) :
1347 Value(Value), Weight(Weight), InsertionOrder(InsertionOrder) {
1348 assert(Weight >= 0 &&
"Weight must be >= 0");
1351 static bool Compare(
const WeightedLeaf &A,
const WeightedLeaf &
B) {
1352 assert(A.Value.getNode() && B.Value.getNode());
1353 return A.Weight == B.Weight ?
1354 (A.InsertionOrder > B.InsertionOrder) :
1355 (A.Weight > B.Weight);
1362 class LeafPrioQueue {
1365 WeightedLeaf ConstElt;
1370 return (!HaveConst && Q.empty());
1374 return Q.size() + HaveConst;
1381 const WeightedLeaf &top() {
1387 WeightedLeaf pop() {
1393 return Q.pop_back_val();
1396 void push(WeightedLeaf L,
bool SeparateConst=
true) {
1397 if (!HaveConst && SeparateConst && isa<ConstantSDNode>(L.Value)) {
1399 cast<ConstantSDNode>(L.Value)->getSExtValue() == 1)
1402 cast<ConstantSDNode>(L.Value)->getSExtValue() == 0)
1415 void pushToBottom(WeightedLeaf L) {
1422 WeightedLeaf findSHL(uint64_t MaxAmount);
1424 WeightedLeaf findMULbyConst();
1426 LeafPrioQueue(
unsigned Opcode) :
1427 HaveConst(
false), Opcode(Opcode) { }
1431 WeightedLeaf LeafPrioQueue::findSHL(uint64_t MaxAmount) {
1433 WeightedLeaf Result;
1435 for (
int Pos = 0,
End = Q.size(); Pos !=
End; ++Pos) {
1436 const WeightedLeaf &L = Q[Pos];
1442 if (!Result.Value.getNode() || Result.Weight > L.Weight ||
1443 (Result.Weight == L.Weight && Result.InsertionOrder > L.InsertionOrder))
1450 if (Result.Value.getNode()) {
1451 Q.erase(&Q[ResultPos]);
1458 WeightedLeaf LeafPrioQueue::findMULbyConst() {
1460 WeightedLeaf Result;
1462 for (
int Pos = 0,
End = Q.size(); Pos !=
End; ++Pos) {
1463 const WeightedLeaf &L = Q[Pos];
1469 if (!Result.Value.getNode() || Result.Weight > L.Weight ||
1470 (Result.Weight == L.Weight && Result.InsertionOrder > L.InsertionOrder))
1477 if (Result.Value.getNode()) {
1478 Q.erase(&Q[ResultPos]);
1485 SDValue HexagonDAGToDAGISel::getMultiplierForSHL(
SDNode *N) {
1487 return CurDAG->getConstant(MulFactor,
SDLoc(N),
1494 unsigned MaxFactor = 0;
1495 for (
int i = 0;
i < 2; ++
i) {
1518 for (
int i = 0;
i < 2; ++
i)
1519 if (isa<ConstantSDNode>(Ops[
i].getNode()) &&
1522 return (NewConst == 1);
1531 SDValue HexagonDAGToDAGISel::factorOutPowerOf2(
SDValue V,
unsigned Power) {
1534 for (
int i=0;
i < 2; ++
i) {
1535 if (isa<ConstantSDNode>(Ops[
i].getNode()) &&
1540 Ops[
i] = CurDAG->getConstant(NewConst,
1547 if (ShiftAmount == Power)
1549 Ops[1] = CurDAG->getConstant(ShiftAmount - Power,
1561 unsigned HexagonDAGToDAGISel::getUsesInFunction(
const Value *V) {
1562 if (GAUsesInFunction.count(V))
1563 return GAUsesInFunction[V];
1565 unsigned Result = 0;
1566 const Function *CurF = CurDAG->getMachineFunction().getFunction();
1568 if (isa<Instruction>(U) &&
1573 GAUsesInFunction[V] = Result;
1583 SDValue HexagonDAGToDAGISel::balanceSubTree(
SDNode *N,
bool TopLevel) {
1584 assert(RootWeights.count(N) &&
"Cannot balance non-root node.");
1585 assert(RootWeights[N] != -2 &&
"This node was RAUW'd!");
1589 if (RootWeights[N] != -1)
1603 Weight = getWeight(balanceSubTree(Op0N).getNode());
1606 Weight = getWeight(Op0N);
1610 Weight += getWeight(balanceSubTree(Op1N).getNode());
1613 Weight += getWeight(Op1N);
1615 RootWeights[
N] = Weight;
1619 DEBUG(
dbgs() <<
"--> No need to balance root (Weight=" << Weight
1620 <<
" Height=" << RootHeights[N] <<
"): ");
1626 DEBUG(
dbgs() <<
"** Balancing root node: ");
1631 LeafPrioQueue Leaves(NOpcode);
1639 bool CanFactorize =
false;
1640 WeightedLeaf Mul1, Mul2;
1641 unsigned MaxPowerOf2 = 0;
1646 bool HaveTopLevelShift =
false;
1652 HaveTopLevelShift =
true;
1656 int InsertionOrder = 0;
1658 bool Imbalanced =
false;
1659 int CurrentWeight = 0;
1660 while (!Worklist.
empty()) {
1663 if (Child.
getNode() != N && RootWeights.count(Child.
getNode())) {
1666 int Weight = RootWeights[Child.
getNode()];
1668 Child = balanceSubTree(Child.
getNode());
1670 Weight = getWeight(Child.
getNode());
1671 }
else if (Weight == -2) {
1675 DEBUG(
dbgs() <<
"--> Subtree was RAUWd. Restarting...\n");
1676 return balanceSubTree(N, TopLevel);
1679 NodeHeights[Child] = 1;
1680 CurrentWeight += Weight;
1683 if (TopLevel && !CanFactorize && !HaveTopLevelShift &&
1689 if (!Mul1.Value.getNode()) {
1690 Mul1 = WeightedLeaf(Child, Weight, InsertionOrder++);
1691 MaxPowerOf2 = PowerOf2;
1693 Mul2 = WeightedLeaf(Child, Weight, InsertionOrder++);
1694 MaxPowerOf2 =
std::min(MaxPowerOf2, PowerOf2);
1697 if (MaxPowerOf2 > 3)
1700 CanFactorize =
true;
1703 Leaves.push(WeightedLeaf(Child, Weight, InsertionOrder++));
1706 int Weight = getWeight(Child.
getNode());
1708 NodeHeights[Child] = getHeight(Child.
getNode());
1709 CurrentWeight += Weight;
1712 GA = WeightedLeaf(Child, Weight, InsertionOrder++);
1714 Leaves.push(WeightedLeaf(Child, Weight, InsertionOrder++));
1718 unsigned ChildOpcode = Child.
getOpcode();
1719 assert(ChildOpcode == NOpcode ||
1725 Op1 = getMultiplierForSHL(Child.
getNode());
1729 if (!NodeHeights.count(Op1) || !NodeHeights.count(Child->
getOperand(0))) {
1730 assert(!NodeHeights.count(Child) &&
"Parent visited before children?");
1740 NodeHeights[Child] = std::max(NodeHeights[Op1],
1747 <<
" weight=" << CurrentWeight <<
" imbalanced="
1748 << Imbalanced <<
"\n");
1754 DEBUG(
dbgs() <<
"--> Found common factor for two MUL children!\n");
1755 int Weight = Mul1.Weight + Mul2.Weight;
1756 int Height = std::max(NodeHeights[Mul1.Value], NodeHeights[Mul2.Value]) + 1;
1757 SDValue Mul1Factored = factorOutPowerOf2(Mul1.Value, MaxPowerOf2);
1758 SDValue Mul2Factored = factorOutPowerOf2(Mul2.Value, MaxPowerOf2);
1760 Mul1Factored, Mul2Factored);
1762 Mul1.Value.getValueType());
1765 NodeHeights[New] = Height;
1766 Leaves.push(WeightedLeaf(New, Weight, Mul1.InsertionOrder));
1767 }
else if (Mul1.Value.getNode()) {
1771 if (Mul2.Value.getNode())
1773 CanFactorize =
false;
1779 bool CombinedGA =
false;
1780 if (NOpcode ==
ISD::ADD && GA.Value.getNode() && Leaves.hasConst() &&
1781 GA.Value.hasOneUse() && N->
use_size() < 3) {
1783 cast<GlobalAddressSDNode>(GA.Value.getOperand(0));
1784 ConstantSDNode *Offset = cast<ConstantSDNode>(Leaves.top().Value);
1786 if (getUsesInFunction(GANode->
getGlobal()) == 1 && Offset->hasOneUse() &&
1787 getTargetLowering()->isOffsetFoldingLegal(GANode)) {
1788 DEBUG(
dbgs() <<
"--> Combining GA and offset (" << Offset->getSExtValue()
1793 CurDAG->getTargetGlobalAddress(GANode->
getGlobal(),
SDLoc(GA.Value),
1795 GANode->
getOffset() + (uint64_t)Offset->getSExtValue());
1796 GA.Value = CurDAG->getNode(GA.Value.getOpcode(),
SDLoc(GA.Value),
1797 GA.Value.getValueType(), NewTGA);
1798 GA.Weight += Leaves.top().Weight;
1800 NodeHeights[GA.Value] = getHeight(GA.Value.getNode());
1809 RootWeights[
N] = CurrentWeight;
1810 RootHeights[
N] = NodeHeights[
SDValue(N, 0)];
1816 if (NOpcode ==
ISD::ADD && GA.Value.getNode()) {
1817 WeightedLeaf
SHL = Leaves.findSHL(31);
1818 if (SHL.Value.getNode()) {
1819 int Height = std::max(NodeHeights[GA.Value], NodeHeights[SHL.Value]) + 1;
1821 GA.Value.getValueType(),
1822 GA.Value, SHL.Value);
1823 GA.Weight = SHL.Weight;
1824 NodeHeights[GA.Value] = Height;
1828 if (GA.Value.getNode())
1833 if (TopLevel && !CanFactorize && Leaves.hasConst()) {
1834 DEBUG(
dbgs() <<
"--> Pushing constant to tip of tree.");
1835 Leaves.pushToBottom(Leaves.pop());
1838 const DataLayout &DL = CurDAG->getDataLayout();
1842 while (Leaves.size() > 1) {
1843 WeightedLeaf L0 = Leaves.pop();
1847 WeightedLeaf L1 = Leaves.findMULbyConst();
1848 if (!L1.Value.getNode())
1851 assert(L0.Weight <= L1.Weight &&
"Priority queue is broken!");
1854 int V0Weight = L0.Weight;
1856 int V1Weight = L1.Weight;
1859 if ((RootWeights.count(V0.
getNode()) && RootWeights[V0.
getNode()] == -2) ||
1860 (RootWeights.count(V1.
getNode()) && RootWeights[V1.
getNode()] == -2)) {
1861 DEBUG(
dbgs() <<
"--> Subtree was RAUWd. Restarting...\n");
1862 return balanceSubTree(N, TopLevel);
1876 assert(NodeHeights.count(V0) && NodeHeights.count(V1) &&
1877 "Children must have been visited before re-combining them!");
1878 int Height = std::max(NodeHeights[V0], NodeHeights[V1]) + 1;
1881 if (V1C && NOpcode ==
ISD::MUL && V1C->getAPIntValue().isPowerOf2())
1882 NewNode = CurDAG->getNode(
1884 CurDAG->getConstant(
1885 V1C->getAPIntValue().logBase2(),
SDLoc(N),
1888 NewNode = CurDAG->getNode(NOpcode,
SDLoc(N), VT, V0, V1);
1890 NodeHeights[NewNode] = Height;
1892 int Weight = V0Weight + V1Weight;
1893 Leaves.push(WeightedLeaf(NewNode, Weight, L0.InsertionOrder));
1895 DEBUG(
dbgs() <<
"--> Built new node (Weight=" << Weight <<
",Height="
1896 << Height <<
"):\n");
1900 assert(Leaves.size() == 1);
1901 SDValue NewRoot = Leaves.top().Value;
1903 assert(NodeHeights.count(NewRoot));
1904 int Height = NodeHeights[NewRoot];
1907 if (NewRoot.getOpcode() ==
ISD::MUL) {
1910 EVT VT = NewRoot.getValueType();
1912 NewRoot = CurDAG->getNode(
1914 CurDAG->getConstant(
1920 if (N != NewRoot.getNode()) {
1922 DEBUG(NewRoot.dump());
1925 CurDAG->ReplaceAllUsesWith(N, NewRoot.getNode());
1927 RootWeights[
N] = -2;
1929 DEBUG(
dbgs() <<
"--> Root unchanged.\n");
1932 RootWeights[NewRoot.getNode()] = Leaves.top().Weight;
1933 RootHeights[NewRoot.getNode()] = Height;
1938 void HexagonDAGToDAGISel::rebalanceAddressTrees() {
1939 for (
auto I = CurDAG->allnodes_begin(),
E = CurDAG->allnodes_end(); I !=
E;) {
1944 SDValue BasePtr = cast<MemSDNode>(
N)->getBasePtr();
1949 if (RootWeights.count(BasePtr.
getNode()))
1952 DEBUG(
dbgs() <<
"** Rebalancing address calculation in node: ");
1961 while (!Worklist.empty()) {
1962 SDNode *N = Worklist.pop_back_val();
1976 if (RootWeights.count(N))
1979 RootWeights[
N] = -1;
1983 RootWeights[BasePtr.
getNode()] = -1;
1987 N = CurDAG->UpdateNodeOperands(N, N->
getOperand(0),
1997 CurDAG->RemoveDeadNodes();
1998 GAUsesInFunction.clear();
1999 RootHeights.clear();
2000 RootWeights.clear();
unsigned getStackAlignment() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd)
Assign this MachineSDNodes's memory reference descriptor list.
bool hasOneUse() const
Return true if there is exactly one use of this node.
static bool willShiftRightEliminate(SDValue V, unsigned Amount)
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
unsigned getStackAlignBaseVReg() const
const GlobalValue * getGlobal() const
static bool doesIntrinsicReturnPredicate(unsigned ID)
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
static unsigned getPowerOf2Factor(SDValue Val)
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Hexagon target-specific information for each MachineFunction.
unsigned getNumOperands() const
Return the number of values used by this operation.
constexpr bool isInt< 16 >(int64_t x)
const SDValue & getOperand(unsigned Num) const
void setNodeId(int Id)
Set unique node id.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const SDValue & getBasePtr() const
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
unsigned getMaxAlignment() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
bool isVector() const
isVector - Return true if this is a vector value type.
A description of a memory reference used in the backend.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static cl::opt< bool > RebalanceOnlyForOptimizations("rebalance-only-opt", cl::Hidden, cl::init(false), cl::desc("Rebalance address tree only if this allows optimizations"))
int64_t getOffset() const
Shift and rotation operations.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
static bool isTargetConstant(const SDValue &V)
APInt bitcastToAPInt() const
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
This class is used to represent EVT's, which are used to parameterize some operations.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
iterator_range< allnodes_iterator > allnodes()
LLVM_NODISCARD bool empty() const
bool getBoolValue() const
Convert APInt to a boolean value.
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const HexagonRegisterInfo & getRegisterInfo() const
HexagonInstrInfo specifics.
Simple integer binary arithmetic operators.
Function Alias Analysis false
const SDValue & getBasePtr() const
bool needsAligna(const MachineFunction &MF) const
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
const APInt & getAPIntValue() const
EVT getMemoryVT() const
Return the type of the in-memory value.
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
This class is used to represent ISD::STORE nodes.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
SDNode * getNode() const
get the SDNode which holds the desired result
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
unsigned getScalarSizeInBits() const
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
initializer< Ty > init(const Ty &Val)
size_t use_size() const
Return the number of uses of this node.
constexpr bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
const SDValue & getOperand(unsigned i) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static ManagedStatic< OptionRegistry > OR
static const unsigned End
unsigned getOpcode() const
FunctionPass class - This class is used to implement most global optimizations.
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
const SDValue & getValue() const
EVT - Extended Value Type.
const APFloat & getValueAPF() const
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
This class contains a discriminated union of information about pointers in memory operands...
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const SDValue & getOffset() const
static bool isOpcodeHandled(const SDNode *N)
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
void dump() const
Dump this node, for debugging.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
unsigned logBase2() const
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
unsigned getObjectAlignment(int ObjectIdx) const
Return the alignment of the specified stack object.
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
An SDNode that represents everything that will be needed to construct a MachineInstr.
const SDValue & getChain() const
LLVM_NODISCARD T pop_back_val()
CHAIN = SC CHAIN, Imm128 - System call.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
This is an abstract virtual class for memory operations.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Class for arbitrary precision integers.
Select(COND, TRUEVAL, FALSEVAL).
int64_t getSExtValue() const
iterator_range< user_iterator > users()
ZERO_EXTEND - Used for integer types, zeroing the new bits.
ANY_EXTEND - Used for integer types. The high bits are undefined.
static cl::opt< bool > RebalanceOnlyImbalancedTrees("rebalance-only-imbal", cl::Hidden, cl::init(false), cl::desc("Rebalance address tree only if it is imbalanced"))
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static cl::opt< bool > EnableAddressRebalancing("isel-rebalance-addr", cl::Hidden, cl::init(true), cl::desc("Rebalance address calculation trees to improve ""instruction selection"))
uint64_t getConstantOperandVal(unsigned i) const
Bitwise operators - logical and, logical or, logical xor.
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
virtual MVT getScalarShiftAmountTy(const DataLayout &, EVT) const
EVT is not used in-tree, but is used by out-of-tree target.
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
APFloat abs(APFloat X)
Returns the absolute value of the argument.
const SDValue & getOffset() const
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
FunctionPass * createHexagonISelDag(HexagonTargetMachine &TM, CodeGenOpt::Level OptLevel)
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
This class is used to form a handle around another node that is persistent and is updated across invo...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool isSimple() const
isSimple - Test if the given EVT is simple (as opposed to being extended).
Module * getParent()
Get the module that this global value is contained inside of...
LLVM Value Representation.
const HexagonInstrInfo * getInstrInfo() const override
bool isTruncatingStore() const
Return true if the op does a truncation before store.
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
static const Function * getParent(const Value *V)
StringRef - Represent a constant reference to a string, i.e.
unsigned getAlignment() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
uint64_t getZExtValue() const
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
This class is used to represent ISD::LOAD nodes.