23#define DEBUG_TYPE "systemz-isel"
24#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
28struct SystemZAddressingMode {
63 bool IncludesDynAlloc;
65 SystemZAddressingMode(AddrForm form, DispRange dr)
66 :
Form(form), DR(dr), Disp(0), IncludesDynAlloc(
false) {}
69 bool hasIndexField() {
return Form != FormBD; }
72 bool isDynAlloc() {
return Form == FormBDXDynAlloc; }
75 errs() <<
"SystemZAddressingMode " <<
this <<
'\n';
79 Base.getNode()->dump(DAG);
83 if (hasIndexField()) {
86 Index.getNode()->dump(DAG);
91 errs() <<
" Disp " << Disp;
93 errs() <<
" + ADJDYNALLOC";
119struct RxSBGOperands {
121 : Opcode(
Op), BitSize(
N.getValueSizeInBits()),
122 Mask(
allOnes(BitSize)), Input(
N), Start(64 - BitSize),
End(63),
139 return CurDAG->getTargetConstant(Imm,
SDLoc(
Node),
Node->getValueType(0));
152 bool expandAddress(SystemZAddressingMode &AM,
bool IsBase)
const;
155 bool selectAddress(
SDValue N, SystemZAddressingMode &AM)
const;
158 void getAddressOperands(
const SystemZAddressingMode &AM,
EVT VT,
160 void getAddressOperands(
const SystemZAddressingMode &AM,
EVT VT,
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR,
SDValue Addr,
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR,
SDValue Addr,
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm
Form,
193 return selectBDAddr(SystemZAddressingMode::Disp12Only,
Addr,
Base, Disp);
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair,
Addr,
Base, Disp);
199 return selectBDAddr(SystemZAddressingMode::Disp20Only,
Addr,
Base, Disp);
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair,
Addr,
Base, Disp);
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair,
Addr,
Base, Disp);
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair,
Addr,
Base, Disp);
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
277 bool refineRxSBGMask(RxSBGOperands &RxSBG,
uint64_t Mask)
const;
281 bool expandRxSBG(RxSBGOperands &RxSBG)
const;
319 bool tryFoldLoadStoreIntoMemOperand(
SDNode *
Node);
334 bool storeLoadCanUseMVC(
SDNode *
N)
const;
339 bool storeLoadCanUseBlockBinary(
SDNode *
N,
unsigned I)
const;
343 bool storeLoadIsAligned(
SDNode *
N)
const;
351 SystemZDAGToDAGISel() =
delete;
358 if (
F.getFnAttribute(
"fentry-call").getValueAsString() !=
"true") {
359 if (
F.hasFnAttribute(
"mnop-mcount"))
361 if (
F.hasFnAttribute(
"mrecord-mcount"))
373 std::vector<SDValue> &OutOps)
override;
378 #include "SystemZGenDAGISel.inc"
382char SystemZDAGToDAGISel::ID = 0;
388 return new SystemZDAGToDAGISel(
TM, OptLevel);
394static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
396 case SystemZAddressingMode::Disp12Only:
397 return isUInt<12>(Val);
399 case SystemZAddressingMode::Disp12Pair:
400 case SystemZAddressingMode::Disp20Only:
401 case SystemZAddressingMode::Disp20Pair:
402 return isInt<20>(Val);
404 case SystemZAddressingMode::Disp20Only128:
405 return isInt<20>(Val) && isInt<20>(Val + 8);
425 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
427 AM.IncludesDynAlloc =
true;
437 if (AM.hasIndexField() && !AM.Index.getNode()) {
447static bool expandDisp(SystemZAddressingMode &AM,
bool IsBase,
450 int64_t TestDisp = AM.Disp + Op1;
462bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
465 unsigned Opcode =
N.getOpcode();
484 cast<ConstantSDNode>(Op0)->getSExtValue());
487 cast<ConstantSDNode>(Op1)->getSExtValue());
497 cast<GlobalAddressSDNode>(Anchor)->getOffset());
505static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
508 case SystemZAddressingMode::Disp12Only:
509 case SystemZAddressingMode::Disp20Only:
510 case SystemZAddressingMode::Disp20Only128:
513 case SystemZAddressingMode::Disp12Pair:
515 return isUInt<12>(Val);
517 case SystemZAddressingMode::Disp20Pair:
519 return !isUInt<12>(Val);
543 if (isUInt<12>(Disp))
548 if (!isInt<16>(Disp))
557 if (
Index->hasOneUse())
562 unsigned IndexOpcode =
Index->getOpcode();
570 if (
Base->hasOneUse())
577bool SystemZDAGToDAGISel::selectAddress(
SDValue Addr,
578 SystemZAddressingMode &AM)
const {
586 cast<ConstantSDNode>(
Addr)->getSExtValue()))
594 while (expandAddress(AM,
true) ||
595 (AM.Index.getNode() && expandAddress(AM,
false)))
599 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
600 !
shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
608 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
621 if (
N->getNodeId() == -1 ||
634void SystemZDAGToDAGISel::getAddressOperands(
const SystemZAddressingMode &AM,
640 Base = CurDAG->getRegister(0, VT);
644 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
645 }
else if (
Base.getValueType() != VT) {
647 assert(VT == MVT::i32 &&
Base.getValueType() == MVT::i64 &&
648 "Unexpected truncation");
656 Disp = CurDAG->getTargetConstant(AM.Disp,
SDLoc(
Base), VT);
659void SystemZDAGToDAGISel::getAddressOperands(
const SystemZAddressingMode &AM,
663 getAddressOperands(AM, VT,
Base, Disp);
666 if (!
Index.getNode())
668 Index = CurDAG->getRegister(0, VT);
671bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
674 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
675 if (!selectAddress(
Addr, AM))
678 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp);
682bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
685 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
686 if (!selectAddress(
Addr, AM) || AM.Index.getNode())
689 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp);
693bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm
Form,
694 SystemZAddressingMode::DispRange DR,
697 SystemZAddressingMode AM(
Form, DR);
698 if (!selectAddress(
Addr, AM))
701 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp,
Index);
710 if (selectBDXAddr12Only(
Addr, Regs[0], Disp, Regs[1]) &&
711 Regs[0].getNode() && Regs[1].getNode()) {
712 for (
unsigned int I = 0;
I < 2; ++
I) {
720 Index.getOperand(1) == Elem) {
729bool SystemZDAGToDAGISel::detectOrAndInsertion(
SDValue &
Op,
737 auto *MaskNode = dyn_cast<ConstantSDNode>(
Op.getOperand(1).getNode());
742 uint64_t AndMask = MaskNode->getZExtValue();
743 if (InsertMask & AndMask)
749 if (Used != (AndMask | InsertMask)) {
750 KnownBits Known = CurDAG->computeKnownBits(
Op.getOperand(0));
755 Op =
Op.getOperand(0);
759bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
762 if (RxSBG.Rotate != 0)
763 Mask = (
Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
765 if (
TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
775 if (RxSBG.Rotate != 0)
776 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
777 return (Mask & RxSBG.Mask) != 0;
780bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG)
const {
782 unsigned Opcode =
N.getOpcode();
785 if (RxSBG.Opcode == SystemZ::RNSBG)
787 uint64_t BitSize =
N.getValueSizeInBits();
789 if (!refineRxSBGMask(RxSBG, Mask))
791 RxSBG.Input =
N.getOperand(0);
795 if (RxSBG.Opcode == SystemZ::RNSBG)
798 auto *MaskNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
804 if (!refineRxSBGMask(RxSBG, Mask)) {
808 KnownBits Known = CurDAG->computeKnownBits(Input);
810 if (!refineRxSBGMask(RxSBG, Mask))
818 if (RxSBG.Opcode != SystemZ::RNSBG)
821 auto *MaskNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
827 if (!refineRxSBGMask(RxSBG, Mask)) {
831 KnownBits Known = CurDAG->computeKnownBits(Input);
832 Mask &= ~Known.One.getZExtValue();
833 if (!refineRxSBGMask(RxSBG, Mask))
842 if (RxSBG.BitSize != 64 ||
N.getValueType() != MVT::i64)
844 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
848 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
849 RxSBG.Input =
N.getOperand(0);
855 RxSBG.Input =
N.getOperand(0);
859 if (RxSBG.Opcode != SystemZ::RNSBG) {
861 unsigned InnerBitSize =
N.getOperand(0).getValueSizeInBits();
862 if (!refineRxSBGMask(RxSBG,
allOnes(InnerBitSize)))
865 RxSBG.Input =
N.getOperand(0);
873 unsigned BitSize =
N.getValueSizeInBits();
874 unsigned InnerBitSize =
N.getOperand(0).getValueSizeInBits();
878 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
879 RxSBG.Rotate += (BitSize - InnerBitSize);
884 RxSBG.Input =
N.getOperand(0);
889 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
893 uint64_t Count = CountNode->getZExtValue();
894 unsigned BitSize =
N.getValueSizeInBits();
895 if (Count < 1 || Count >= BitSize)
898 if (RxSBG.Opcode == SystemZ::RNSBG) {
905 if (!refineRxSBGMask(RxSBG,
allOnes(BitSize - Count) << Count))
909 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
910 RxSBG.Input =
N.getOperand(0);
916 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
920 uint64_t Count = CountNode->getZExtValue();
921 unsigned BitSize =
N.getValueSizeInBits();
922 if (Count < 1 || Count >= BitSize)
933 if (!refineRxSBGMask(RxSBG,
allOnes(BitSize - Count)))
937 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
938 RxSBG.Input =
N.getOperand(0);
947 SDNode *
N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, VT);
953 if (
N.getValueType() == MVT::i32 && VT == MVT::i64)
954 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
955 DL, VT, getUNDEF(
DL, MVT::i64),
N);
956 if (
N.getValueType() == MVT::i64 && VT == MVT::i32)
957 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32,
DL, VT,
N);
958 assert(
N.getValueType() == VT &&
"Unexpected value types");
962bool SystemZDAGToDAGISel::tryRISBGZero(
SDNode *
N) {
964 EVT VT =
N->getValueType(0);
967 RxSBGOperands RISBG(SystemZ::RISBG,
SDValue(
N, 0));
969 while (expandRxSBG(RISBG))
976 if (Count == 0 || isa<ConstantSDNode>(RISBG.Input))
981 if (Count == 1 &&
N->getOpcode() !=
ISD::AND)
987 if (RISBG.Rotate == 0) {
988 bool PreferAnd =
false;
994 else if (RISBG.Mask == 0xff ||
995 RISBG.Mask == 0xffff ||
996 RISBG.Mask == 0x7fffffff ||
1002 else if (
auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
1003 if (
Load->getMemoryVT() == MVT::i32 &&
1006 RISBG.Mask == 0xffffff00 &&
1007 Subtarget->hasLoadAndZeroRightmostByte())
1017 if (
N !=
New.getNode()) {
1020 ReplaceNode(
N,
New.getNode());
1024 if (!
N->isMachineOpcode())
1030 unsigned Opcode = SystemZ::RISBG;
1032 if (Subtarget->hasMiscellaneousExtensions())
1033 Opcode = SystemZ::RISBGN;
1034 EVT OpcodeVT = MVT::i64;
1035 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1040 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1041 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1042 ((RISBG.End + RISBG.Rotate) & 63) >=
1043 ((RISBG.Start + RISBG.Rotate) & 63)) {
1044 Opcode = SystemZ::RISBMux;
1045 OpcodeVT = MVT::i32;
1050 getUNDEF(
DL, OpcodeVT),
1051 convertTo(
DL, OpcodeVT, RISBG.Input),
1052 CurDAG->getTargetConstant(RISBG.Start,
DL, MVT::i32),
1053 CurDAG->getTargetConstant(RISBG.End | 128,
DL, MVT::i32),
1054 CurDAG->getTargetConstant(RISBG.Rotate,
DL, MVT::i32)
1058 ReplaceNode(
N,
New.getNode());
1062bool SystemZDAGToDAGISel::tryRxSBG(
SDNode *
N,
unsigned Opcode) {
1064 EVT VT =
N->getValueType(0);
1069 RxSBGOperands RxSBG[] = {
1070 RxSBGOperands(
Opcode,
N->getOperand(0)),
1071 RxSBGOperands(
Opcode,
N->getOperand(1))
1073 unsigned Count[] = { 0, 0 };
1074 for (
unsigned I = 0;
I < 2; ++
I)
1075 while (RxSBG[
I].Input->
hasOneUse() && expandRxSBG(RxSBG[
I]))
1087 if (Count[0] == 0 && Count[1] == 0)
1091 unsigned I = Count[0] > Count[1] ? 0 : 1;
1095 if (
Opcode == SystemZ::ROSBG && (RxSBG[
I].Mask & 0xff) == 0)
1096 if (
auto *Load = dyn_cast<LoadSDNode>(Op0.
getNode()))
1097 if (
Load->getMemoryVT() == MVT::i8)
1102 if (
Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[
I].Mask)) {
1105 if (Subtarget->hasMiscellaneousExtensions())
1106 Opcode = SystemZ::RISBGN;
1110 convertTo(
DL, MVT::i64, Op0),
1111 convertTo(
DL, MVT::i64, RxSBG[
I].Input),
1112 CurDAG->getTargetConstant(RxSBG[
I].Start,
DL, MVT::i32),
1113 CurDAG->getTargetConstant(RxSBG[
I].
End,
DL, MVT::i32),
1114 CurDAG->getTargetConstant(RxSBG[
I].Rotate,
DL, MVT::i32)
1118 ReplaceNode(
N,
New.getNode());
1122void SystemZDAGToDAGISel::splitLargeImmediate(
unsigned Opcode,
SDNode *
Node,
1125 EVT VT =
Node->getValueType(0);
1145 SelectCode(
Upper.getNode());
1146 Upper = Handle.getValue();
1152 ReplaceNode(
Node,
Or.getNode());
1154 SelectCode(
Or.getNode());
1157void SystemZDAGToDAGISel::loadVectorConstant(
1164 EVT VT =
Node->getValueType(0);
1167 for (
unsigned OpVal : VCI.
OpVals)
1168 Ops.
push_back(CurDAG->getTargetConstant(OpVal,
DL, MVT::i32));
1172 ReplaceNode(
Node,
Op.getNode());
1176 SelectCode(BitCast.
getNode());
1178 unsigned SubRegIdx =
1179 (VT.
getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1181 Node, CurDAG->getTargetExtractSubreg(SubRegIdx,
DL, VT,
Op).getNode());
1183 SelectCode(
Op.getNode());
1186bool SystemZDAGToDAGISel::tryGather(
SDNode *
N,
unsigned Opcode) {
1188 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1192 unsigned Elem = ElemN->getZExtValue();
1193 EVT VT =
N->getValueType(0);
1197 auto *
Load = dyn_cast<LoadSDNode>(
N->getOperand(1));
1198 if (!Load || !
Load->hasNUsesOfValue(1, 0))
1200 if (
Load->getMemoryVT().getSizeInBits() !=
1201 Load->getValueType(0).getSizeInBits())
1205 if (!selectBDVAddr12Only(
Load->getBasePtr(), ElemV,
Base, Disp,
Index) ||
1212 CurDAG->getTargetConstant(Elem,
DL, MVT::i32),
Load->getChain()
1214 SDNode *Res = CurDAG->getMachineNode(
Opcode,
DL, VT, MVT::Other, Ops);
1216 ReplaceNode(
N, Res);
1224 if (
Store->getMemoryVT().getSizeInBits() !=
Value.getValueSizeInBits())
1228 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1234 unsigned Elem = ElemN->getZExtValue();
1239 if (!selectBDVAddr12Only(
Store->getBasePtr(), ElemV,
Base, Disp,
Index) ||
1245 Vec,
Base, Disp,
Index, CurDAG->getTargetConstant(Elem,
DL, MVT::i32),
1248 ReplaceNode(Store, CurDAG->getMachineNode(
Opcode,
DL, MVT::Other, Ops));
1276 LoadNode = cast<LoadSDNode>(Load);
1279 if (!Load.hasOneUse())
1291 bool ChainCheck =
false;
1292 if (Chain == Load.getValue(1)) {
1299 const unsigned int Max = 1024;
1302 if (
Op == Load.getValue(1)) {
1315 if (
Op.getNode() != LoadNode)
1326 MVT::Other, ChainOps);
1343bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(
SDNode *
Node) {
1353 unsigned NewOpc = 0;
1354 bool NegateOperand =
false;
1359 NegateOperand =
true;
1362 if (MemVT == MVT::i32)
1363 NewOpc = SystemZ::ASI;
1364 else if (MemVT == MVT::i64)
1365 NewOpc = SystemZ::AGSI;
1370 NegateOperand =
true;
1373 if (MemVT == MVT::i32)
1374 NewOpc = SystemZ::ALSI;
1375 else if (MemVT == MVT::i64)
1376 NewOpc = SystemZ::ALGSI;
1389 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1392 auto OperandV = OperandC->getAPIntValue();
1394 OperandV = -OperandV;
1395 if (OperandV.getSignificantBits() > 8)
1397 Operand = CurDAG->getTargetConstant(OperandV,
DL, MemVT);
1403 SDValue Ops[] = {
Base, Disp, Operand, InputChain };
1405 CurDAG->getMachineNode(NewOpc,
DL, MVT::i32, MVT::Other, Ops);
1406 CurDAG->setNodeMemRefs(
1411 CurDAG->RemoveDeadNode(
Node);
1415bool SystemZDAGToDAGISel::canUseBlockOperation(
StoreSDNode *Store,
1418 if (
Load->getMemoryVT() !=
Store->getMemoryVT())
1422 if (
Load->isVolatile() ||
Store->isVolatile())
1426 if (
Load->isInvariant() &&
Load->isDereferenceable())
1430 const Value *V1 =
Load->getMemOperand()->getValue();
1437 int64_t End1 =
Load->getSrcValueOffset() +
Size;
1438 int64_t End2 =
Store->getSrcValueOffset() +
Size;
1439 if (V1 == V2 && End1 == End2)
1446bool SystemZDAGToDAGISel::storeLoadCanUseMVC(
SDNode *
N)
const {
1447 auto *
Store = cast<StoreSDNode>(
N);
1448 auto *
Load = cast<LoadSDNode>(
Store->getValue());
1462 return canUseBlockOperation(Store, Load);
1465bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(
SDNode *
N,
1467 auto *StoreA = cast<StoreSDNode>(
N);
1468 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 -
I));
1469 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(
I));
1470 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1471 canUseBlockOperation(StoreA, LoadB);
1474bool SystemZDAGToDAGISel::storeLoadIsAligned(
SDNode *
N)
const {
1476 auto *MemAccess = cast<LSBaseSDNode>(
N);
1477 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1480 assert(MMO &&
"Expected a memory operand.");
1483 if (MemAccess->getAlign().value() < StoreSize ||
1484 !MemAccess->getOffset().isUndef())
1493 if ((PSV->isGOT() || PSV->isConstantPool()))
1499 dyn_cast<GlobalAddressSDNode>(
BasePtr.getOperand(0))) {
1501 if (GA->getOffset() % StoreSize != 0)
1514void SystemZDAGToDAGISel::Select(
SDNode *
Node) {
1516 if (
Node->isMachineOpcode()) {
1518 Node->setNodeId(-1);
1526 if (tryRxSBG(
Node, SystemZ::ROSBG))
1532 if (tryRxSBG(
Node, SystemZ::RXSBG))
1539 if (
Node->getValueType(0) == MVT::i64 &&
1541 if (
auto *Op1 = dyn_cast<ConstantSDNode>(
Node->getOperand(1))) {
1542 uint64_t Val = Op1->getZExtValue();
1545 if (Subtarget->hasMiscellaneousExtensions3()) {
1546 unsigned ChildOpcode =
Node->getOperand(0).getOpcode();
1555 auto Op0 =
Node->getOperand(0);
1556 if (
auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->
getOperand(1)))
1557 if (Op0Op1->getZExtValue() == (
uint64_t)-1)
1571 if (tryRxSBG(
Node, SystemZ::RNSBG))
1578 if (tryRISBGZero(
Node))
1585 if (
Node->getValueType(0) == MVT::i64) {
1586 uint64_t Val = cast<ConstantSDNode>(
Node)->getZExtValue();
1601 (Subtarget->hasLoadStoreOnCond2() &&
1602 Node->getValueType(0).isInteger() &&
1604 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1606 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1610 cast<ConstantSDNode>(CCValid.
getNode())->getZExtValue();
1612 cast<ConstantSDNode>(CCMask.
getNode())->getZExtValue();
1614 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1618 CurDAG->UpdateNodeOperands(
Node, Op1, Op0, CCValid, CCMask, Op4);
1619 if (UpdatedNode !=
Node) {
1621 ReplaceNode(
Node, UpdatedNode);
1629 EVT VT =
Node->getValueType(0);
1631 if (ElemBitSize == 32) {
1632 if (tryGather(
Node, SystemZ::VGEF))
1634 }
else if (ElemBitSize == 64) {
1635 if (tryGather(
Node, SystemZ::VGEG))
1642 auto *BVN = cast<BuildVectorSDNode>(
Node);
1645 loadVectorConstant(VCI,
Node);
1653 if (
Imm.isZero() ||
Imm.isNegZero())
1658 loadVectorConstant(VCI,
Node);
1663 if (tryFoldLoadStoreIntoMemOperand(
Node))
1666 unsigned ElemBitSize =
Store->getValue().getValueSizeInBits();
1667 if (ElemBitSize == 32) {
1668 if (tryScatter(Store, SystemZ::VSCEF))
1670 }
else if (ElemBitSize == 64) {
1671 if (tryScatter(Store, SystemZ::VSCEG))
1681bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand(
1683 std::vector<SDValue> &OutOps) {
1684 SystemZAddressingMode::AddrForm
Form;
1685 SystemZAddressingMode::DispRange DispRange;
1688 switch(ConstraintID) {
1691 case InlineAsm::ConstraintCode::i:
1692 case InlineAsm::ConstraintCode::Q:
1693 case InlineAsm::ConstraintCode::ZQ:
1695 Form = SystemZAddressingMode::FormBD;
1696 DispRange = SystemZAddressingMode::Disp12Only;
1698 case InlineAsm::ConstraintCode::R:
1699 case InlineAsm::ConstraintCode::ZR:
1701 Form = SystemZAddressingMode::FormBDXNormal;
1702 DispRange = SystemZAddressingMode::Disp12Only;
1704 case InlineAsm::ConstraintCode::S:
1705 case InlineAsm::ConstraintCode::ZS:
1707 Form = SystemZAddressingMode::FormBD;
1708 DispRange = SystemZAddressingMode::Disp20Only;
1710 case InlineAsm::ConstraintCode::T:
1711 case InlineAsm::ConstraintCode::m:
1712 case InlineAsm::ConstraintCode::o:
1713 case InlineAsm::ConstraintCode::p:
1714 case InlineAsm::ConstraintCode::ZT:
1719 Form = SystemZAddressingMode::FormBDXNormal;
1720 DispRange = SystemZAddressingMode::Disp20Only;
1726 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1728 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(),
DL, MVT::i32);
1735 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1743 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1748 OutOps.push_back(
Base);
1749 OutOps.push_back(Disp);
1750 OutOps.push_back(
Index);
1765 if (!
N.hasOneUse() || !
U->hasOneUse())
1772 SDNode *CCUser = *
U->use_begin();
1773 SDNode *CCRegUser =
nullptr;
1775 cast<RegisterSDNode>(CCUser->
getOperand(1))->getReg() == SystemZ::CC) {
1776 for (
auto *U : CCUser->
uses()) {
1777 if (CCRegUser ==
nullptr)
1779 else if (CCRegUser != U)
1783 if (CCRegUser ==
nullptr)
1796 if (!IsLegalToFold(
N, U, CCRegUser, OptLevel,
false))
1806struct IPMConversion {
1807 IPMConversion(
unsigned xorValue, int64_t addValue,
unsigned bit)
1808 : XORValue(xorValue), AddValue(addValue),
Bit(bit) {}
1884 auto *TrueOp = dyn_cast<ConstantSDNode>(
Node->getOperand(0));
1885 auto *FalseOp = dyn_cast<ConstantSDNode>(
Node->getOperand(1));
1886 if (!TrueOp || !FalseOp)
1888 if (FalseOp->getZExtValue() != 0)
1890 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
1893 auto *CCValidOp = dyn_cast<ConstantSDNode>(
Node->getOperand(2));
1894 auto *CCMaskOp = dyn_cast<ConstantSDNode>(
Node->getOperand(3));
1895 if (!CCValidOp || !CCMaskOp)
1897 int CCValid = CCValidOp->getZExtValue();
1898 int CCMask = CCMaskOp->getZExtValue();
1907 CurDAG->getConstant(
IPM.XORValue,
DL, MVT::i32));
1911 CurDAG->getConstant(
IPM.AddValue,
DL, MVT::i32));
1913 EVT VT =
Node->getValueType(0);
1914 if (VT == MVT::i32 &&
IPM.Bit == 31) {
1916 Result = CurDAG->getNode(ShiftOp,
DL, MVT::i32, Result,
1917 CurDAG->getConstant(
IPM.Bit,
DL, MVT::i32));
1922 if (TrueOp->getSExtValue() == 1) {
1925 CurDAG->getConstant(
IPM.Bit,
DL, MVT::i32));
1927 CurDAG->getConstant(1,
DL, VT));
1933 CurDAG->getConstant(ShlAmt,
DL, MVT::i32));
1935 CurDAG->getConstant(SraAmt,
DL, MVT::i32));
1942void SystemZDAGToDAGISel::PreprocessISelDAG() {
1945 if (Subtarget->hasLoadStoreOnCond2())
1948 bool MadeChange =
false;
1951 E = CurDAG->allnodes_end();
1958 switch (
N->getOpcode()) {
1961 Res = expandSelectBoolean(
N);
1966 LLVM_DEBUG(
dbgs() <<
"SystemZ DAG preprocessing replacing:\nOld: ");
1972 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1978 CurDAG->RemoveDeadNodes();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
const HexagonInstrInfo * TII
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask)
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
static uint64_t allOnes(unsigned int Count)
static constexpr uint32_t Opcode
DEMANGLE_DUMP_METHOD void dump() const
uint64_t getZExtValue() const
Get zero extended value.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
Module * getParent()
Get the module that this global value is contained inside of...
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
An SDNode that represents everything that will be needed to construct a MachineInstr.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
bool isNonTemporal() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Special value supplied for machine level alias analysis.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
static int getUninvalidatedNodeId(SDNode *N)
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SystemZInstrInfo * getInstrInfo() const override
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
LLVM Value Representation.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SHL
Shift and rotation operations.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isPCREL(unsigned Opcode)
static bool isImmHF(uint64_t Val)
static bool isImmLF(uint64_t Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CodeGenOptLevel
Code generation optimization level.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Or
Bitwise or logical OR of integers.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)