23#define DEBUG_TYPE "systemz-isel"
24#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
28struct SystemZAddressingMode {
63 bool IncludesDynAlloc;
65 SystemZAddressingMode(AddrForm form, DispRange dr)
66 :
Form(form), DR(dr), Disp(0), IncludesDynAlloc(
false) {}
69 bool hasIndexField() {
return Form != FormBD; }
72 bool isDynAlloc() {
return Form == FormBDXDynAlloc; }
75 errs() <<
"SystemZAddressingMode " <<
this <<
'\n';
79 Base.getNode()->dump(DAG);
83 if (hasIndexField()) {
86 Index.getNode()->dump(DAG);
91 errs() <<
" Disp " << Disp;
93 errs() <<
" + ADJDYNALLOC";
119struct RxSBGOperands {
121 : Opcode(
Op), BitSize(
N.getValueSizeInBits()),
122 Mask(
allOnes(BitSize)), Input(
N), Start(64 - BitSize),
End(63),
139 return CurDAG->getTargetConstant(Imm,
SDLoc(
Node),
Node->getValueType(0));
152 bool expandAddress(SystemZAddressingMode &AM,
bool IsBase)
const;
155 bool selectAddress(
SDValue N, SystemZAddressingMode &AM)
const;
158 void getAddressOperands(
const SystemZAddressingMode &AM,
EVT VT,
160 void getAddressOperands(
const SystemZAddressingMode &AM,
EVT VT,
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR,
SDValue Addr,
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR,
SDValue Addr,
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm
Form,
193 return selectBDAddr(SystemZAddressingMode::Disp12Only,
Addr,
Base, Disp);
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair,
Addr,
Base, Disp);
199 return selectBDAddr(SystemZAddressingMode::Disp20Only,
Addr,
Base, Disp);
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair,
Addr,
Base, Disp);
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair,
Addr,
Base, Disp);
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair,
Addr,
Base, Disp);
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
277 bool refineRxSBGMask(RxSBGOperands &RxSBG,
uint64_t Mask)
const;
281 bool expandRxSBG(RxSBGOperands &RxSBG)
const;
295 bool tryRxSBG(
SDNode *
N,
unsigned Opcode);
313 bool tryGather(
SDNode *
N,
unsigned Opcode);
316 bool tryScatter(
StoreSDNode *Store,
unsigned Opcode);
321 bool tryFoldLoadStoreIntoMemOperand(
SDNode *
Node);
336 bool storeLoadCanUseMVC(
SDNode *
N)
const;
341 bool storeLoadCanUseBlockBinary(
SDNode *
N,
unsigned I)
const;
345 bool storeLoadIsAligned(
SDNode *
N)
const;
356 SystemZDAGToDAGISel() =
delete;
363 if (
F.getFnAttribute(
"fentry-call").getValueAsString() !=
"true") {
364 if (
F.hasFnAttribute(
"mnop-mcount"))
366 if (
F.hasFnAttribute(
"mrecord-mcount"))
378 std::vector<SDValue> &OutOps)
override;
383 #include "SystemZGenDAGISel.inc"
387char SystemZDAGToDAGISel::ID = 0;
393 return new SystemZDAGToDAGISel(
TM, OptLevel);
399static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
401 case SystemZAddressingMode::Disp12Only:
402 return isUInt<12>(Val);
404 case SystemZAddressingMode::Disp12Pair:
405 case SystemZAddressingMode::Disp20Only:
406 case SystemZAddressingMode::Disp20Pair:
407 return isInt<20>(Val);
409 case SystemZAddressingMode::Disp20Only128:
410 return isInt<20>(Val) && isInt<20>(Val + 8);
430 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
432 AM.IncludesDynAlloc =
true;
442 if (AM.hasIndexField() && !AM.Index.getNode()) {
452static bool expandDisp(SystemZAddressingMode &AM,
bool IsBase,
455 int64_t TestDisp = AM.Disp + Op1;
467bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
470 unsigned Opcode =
N.getOpcode();
472 if (Opcode ==
ISD::TRUNCATE &&
N.getOperand(0).getValueSizeInBits() <= 64) {
474 Opcode =
N.getOpcode();
476 if (Opcode ==
ISD::ADD || CurDAG->isBaseWithConstantOffset(
N)) {
490 cast<ConstantSDNode>(Op0)->getSExtValue());
493 cast<ConstantSDNode>(Op1)->getSExtValue());
503 cast<GlobalAddressSDNode>(Anchor)->getOffset());
511static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
514 case SystemZAddressingMode::Disp12Only:
515 case SystemZAddressingMode::Disp20Only:
516 case SystemZAddressingMode::Disp20Only128:
519 case SystemZAddressingMode::Disp12Pair:
521 return isUInt<12>(Val);
523 case SystemZAddressingMode::Disp20Pair:
525 return !isUInt<12>(Val);
549 if (isUInt<12>(Disp))
554 if (!isInt<16>(Disp))
563 if (
Index->hasOneUse())
568 unsigned IndexOpcode =
Index->getOpcode();
576 if (
Base->hasOneUse())
583bool SystemZDAGToDAGISel::selectAddress(
SDValue Addr,
584 SystemZAddressingMode &AM)
const {
592 cast<ConstantSDNode>(
Addr)->getSExtValue()))
600 while (expandAddress(AM,
true) ||
601 (AM.Index.getNode() && expandAddress(AM,
false)))
605 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
606 !
shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
614 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
627 if (
N->getNodeId() == -1 ||
640void SystemZDAGToDAGISel::getAddressOperands(
const SystemZAddressingMode &AM,
646 Base = CurDAG->getRegister(0, VT);
650 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
651 }
else if (
Base.getValueType() != VT) {
653 assert(VT == MVT::i32 &&
Base.getValueType() == MVT::i64 &&
654 "Unexpected truncation");
662 Disp = CurDAG->getTargetConstant(AM.Disp,
SDLoc(
Base), VT);
665void SystemZDAGToDAGISel::getAddressOperands(
const SystemZAddressingMode &AM,
669 getAddressOperands(AM, VT,
Base, Disp);
672 if (!
Index.getNode())
674 Index = CurDAG->getRegister(0, VT);
677bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
680 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
681 if (!selectAddress(
Addr, AM))
684 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp);
688bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
691 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
692 if (!selectAddress(
Addr, AM) || AM.Index.getNode())
695 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp);
699bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm
Form,
700 SystemZAddressingMode::DispRange DR,
703 SystemZAddressingMode AM(
Form, DR);
704 if (!selectAddress(
Addr, AM))
707 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp,
Index);
716 if (selectBDXAddr12Only(
Addr, Regs[0], Disp, Regs[1]) &&
717 Regs[0].getNode() && Regs[1].getNode()) {
718 for (
unsigned int I = 0;
I < 2; ++
I) {
726 Index.getOperand(1) == Elem) {
735bool SystemZDAGToDAGISel::detectOrAndInsertion(
SDValue &
Op,
743 auto *MaskNode = dyn_cast<ConstantSDNode>(
Op.getOperand(1).getNode());
748 uint64_t AndMask = MaskNode->getZExtValue();
749 if (InsertMask & AndMask)
755 if (Used != (AndMask | InsertMask)) {
756 KnownBits Known = CurDAG->computeKnownBits(
Op.getOperand(0));
761 Op =
Op.getOperand(0);
765bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
768 if (RxSBG.Rotate != 0)
769 Mask = (
Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
771 if (
TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
781 if (RxSBG.Rotate != 0)
782 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
783 return (Mask & RxSBG.Mask) != 0;
786bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG)
const {
788 unsigned Opcode =
N.getOpcode();
791 if (RxSBG.Opcode == SystemZ::RNSBG)
793 if (
N.getOperand(0).getValueSizeInBits() > 64)
795 uint64_t BitSize =
N.getValueSizeInBits();
797 if (!refineRxSBGMask(RxSBG, Mask))
799 RxSBG.Input =
N.getOperand(0);
803 if (RxSBG.Opcode == SystemZ::RNSBG)
806 auto *MaskNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
812 if (!refineRxSBGMask(RxSBG, Mask)) {
816 KnownBits Known = CurDAG->computeKnownBits(Input);
818 if (!refineRxSBGMask(RxSBG, Mask))
826 if (RxSBG.Opcode != SystemZ::RNSBG)
829 auto *MaskNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
835 if (!refineRxSBGMask(RxSBG, Mask)) {
839 KnownBits Known = CurDAG->computeKnownBits(Input);
840 Mask &= ~Known.One.getZExtValue();
841 if (!refineRxSBGMask(RxSBG, Mask))
850 if (RxSBG.BitSize != 64 ||
N.getValueType() != MVT::i64)
852 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
856 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
857 RxSBG.Input =
N.getOperand(0);
863 RxSBG.Input =
N.getOperand(0);
867 if (RxSBG.Opcode != SystemZ::RNSBG) {
869 unsigned InnerBitSize =
N.getOperand(0).getValueSizeInBits();
870 if (!refineRxSBGMask(RxSBG,
allOnes(InnerBitSize)))
873 RxSBG.Input =
N.getOperand(0);
881 unsigned BitSize =
N.getValueSizeInBits();
882 unsigned InnerBitSize =
N.getOperand(0).getValueSizeInBits();
886 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
887 RxSBG.Rotate += (BitSize - InnerBitSize);
892 RxSBG.Input =
N.getOperand(0);
897 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
901 uint64_t Count = CountNode->getZExtValue();
902 unsigned BitSize =
N.getValueSizeInBits();
903 if (Count < 1 || Count >= BitSize)
906 if (RxSBG.Opcode == SystemZ::RNSBG) {
913 if (!refineRxSBGMask(RxSBG,
allOnes(BitSize - Count) << Count))
917 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
918 RxSBG.Input =
N.getOperand(0);
924 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
928 uint64_t Count = CountNode->getZExtValue();
929 unsigned BitSize =
N.getValueSizeInBits();
930 if (Count < 1 || Count >= BitSize)
933 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode ==
ISD::SRA) {
941 if (!refineRxSBGMask(RxSBG,
allOnes(BitSize - Count)))
945 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
946 RxSBG.Input =
N.getOperand(0);
955 SDNode *
N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, VT);
961 if (
N.getValueType() == MVT::i32 && VT == MVT::i64)
962 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
963 DL, VT, getUNDEF(
DL, MVT::i64),
N);
964 if (
N.getValueType() == MVT::i64 && VT == MVT::i32)
965 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32,
DL, VT,
N);
966 assert(
N.getValueType() == VT &&
"Unexpected value types");
970bool SystemZDAGToDAGISel::tryRISBGZero(
SDNode *
N) {
972 EVT VT =
N->getValueType(0);
975 RxSBGOperands RISBG(SystemZ::RISBG,
SDValue(
N, 0));
977 while (expandRxSBG(RISBG))
984 if (Count == 0 || isa<ConstantSDNode>(RISBG.Input))
989 if (Count == 1 &&
N->getOpcode() !=
ISD::AND)
995 if (RISBG.Rotate == 0) {
996 bool PreferAnd =
false;
1002 else if (RISBG.Mask == 0xff ||
1003 RISBG.Mask == 0xffff ||
1004 RISBG.Mask == 0x7fffffff ||
1010 else if (
auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
1011 if (
Load->getMemoryVT() == MVT::i32 &&
1014 RISBG.Mask == 0xffffff00 &&
1015 Subtarget->hasLoadAndZeroRightmostByte())
1025 if (
N !=
New.getNode()) {
1028 ReplaceNode(
N,
New.getNode());
1032 if (!
N->isMachineOpcode())
1038 unsigned Opcode = SystemZ::RISBG;
1040 if (Subtarget->hasMiscellaneousExtensions())
1041 Opcode = SystemZ::RISBGN;
1042 EVT OpcodeVT = MVT::i64;
1043 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1048 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1049 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1050 ((RISBG.End + RISBG.Rotate) & 63) >=
1051 ((RISBG.Start + RISBG.Rotate) & 63)) {
1052 Opcode = SystemZ::RISBMux;
1053 OpcodeVT = MVT::i32;
1058 getUNDEF(
DL, OpcodeVT),
1059 convertTo(
DL, OpcodeVT, RISBG.Input),
1060 CurDAG->getTargetConstant(RISBG.Start,
DL, MVT::i32),
1061 CurDAG->getTargetConstant(RISBG.End | 128,
DL, MVT::i32),
1062 CurDAG->getTargetConstant(RISBG.Rotate,
DL, MVT::i32)
1065 DL, VT,
SDValue(CurDAG->getMachineNode(Opcode,
DL, OpcodeVT, Ops), 0));
1066 ReplaceNode(
N,
New.getNode());
1070bool SystemZDAGToDAGISel::tryRxSBG(
SDNode *
N,
unsigned Opcode) {
1072 EVT VT =
N->getValueType(0);
1077 RxSBGOperands RxSBG[] = {
1078 RxSBGOperands(Opcode,
N->getOperand(0)),
1079 RxSBGOperands(Opcode,
N->getOperand(1))
1081 unsigned Count[] = { 0, 0 };
1082 for (
unsigned I = 0;
I < 2; ++
I)
1083 while (RxSBG[
I].Input->
hasOneUse() && expandRxSBG(RxSBG[
I]))
1095 if (Count[0] == 0 && Count[1] == 0)
1099 unsigned I = Count[0] > Count[1] ? 0 : 1;
1103 if (Opcode == SystemZ::ROSBG && (RxSBG[
I].Mask & 0xff) == 0)
1104 if (
auto *Load = dyn_cast<LoadSDNode>(Op0.
getNode()))
1105 if (
Load->getMemoryVT() == MVT::i8)
1110 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[
I].Mask)) {
1111 Opcode = SystemZ::RISBG;
1113 if (Subtarget->hasMiscellaneousExtensions())
1114 Opcode = SystemZ::RISBGN;
1118 convertTo(
DL, MVT::i64, Op0),
1119 convertTo(
DL, MVT::i64, RxSBG[
I].Input),
1120 CurDAG->getTargetConstant(RxSBG[
I].Start,
DL, MVT::i32),
1121 CurDAG->getTargetConstant(RxSBG[
I].
End,
DL, MVT::i32),
1122 CurDAG->getTargetConstant(RxSBG[
I].Rotate,
DL, MVT::i32)
1125 DL, VT,
SDValue(CurDAG->getMachineNode(Opcode,
DL, MVT::i64, Ops), 0));
1126 ReplaceNode(
N,
New.getNode());
1130void SystemZDAGToDAGISel::splitLargeImmediate(
unsigned Opcode,
SDNode *
Node,
1133 EVT VT =
Node->getValueType(0);
1153 SelectCode(
Upper.getNode());
1154 Upper = Handle.getValue();
1160 ReplaceNode(
Node,
Or.getNode());
1162 SelectCode(
Or.getNode());
1165void SystemZDAGToDAGISel::loadVectorConstant(
1172 EVT VT =
Node->getValueType(0);
1175 for (
unsigned OpVal : VCI.
OpVals)
1176 Ops.
push_back(CurDAG->getTargetConstant(OpVal,
DL, MVT::i32));
1180 ReplaceNode(
Node,
Op.getNode());
1184 SelectCode(BitCast.
getNode());
1186 unsigned SubRegIdx =
1187 (VT.
getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1189 Node, CurDAG->getTargetExtractSubreg(SubRegIdx,
DL, VT,
Op).getNode());
1191 SelectCode(
Op.getNode());
1198 SDValue CP = CurDAG->getTargetConstantPool(
1200 TLI->getPointerTy(CurDAG->getDataLayout()));
1202 EVT PtrVT =
CP.getValueType();
1204 SDValue(CurDAG->getMachineNode(SystemZ::LARL,
DL, PtrVT, CP), 0),
1205 CurDAG->getTargetConstant(0,
DL, PtrVT),
1206 CurDAG->getRegister(0, PtrVT),
1207 CurDAG->getEntryNode()
1209 ResNode = CurDAG->getMachineNode(SystemZ::VL,
DL, VT, MVT::Other, Ops);
1219 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
1223bool SystemZDAGToDAGISel::tryGather(
SDNode *
N,
unsigned Opcode) {
1225 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1229 unsigned Elem = ElemN->getZExtValue();
1230 EVT VT =
N->getValueType(0);
1234 auto *
Load = dyn_cast<LoadSDNode>(
N->getOperand(1));
1235 if (!Load || !
Load->hasNUsesOfValue(1, 0))
1237 if (
Load->getMemoryVT().getSizeInBits() !=
1238 Load->getValueType(0).getSizeInBits())
1242 if (!selectBDVAddr12Only(
Load->getBasePtr(), ElemV,
Base, Disp,
Index) ||
1249 CurDAG->getTargetConstant(Elem,
DL, MVT::i32),
Load->getChain()
1251 SDNode *Res = CurDAG->getMachineNode(Opcode,
DL, VT, MVT::Other, Ops);
1253 ReplaceNode(
N, Res);
1257bool SystemZDAGToDAGISel::tryScatter(
StoreSDNode *Store,
unsigned Opcode) {
1261 if (
Store->getMemoryVT().getSizeInBits() !=
Value.getValueSizeInBits())
1265 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1271 unsigned Elem = ElemN->getZExtValue();
1276 if (!selectBDVAddr12Only(
Store->getBasePtr(), ElemV,
Base, Disp,
Index) ||
1282 Vec,
Base, Disp,
Index, CurDAG->getTargetConstant(Elem,
DL, MVT::i32),
1285 ReplaceNode(Store, CurDAG->getMachineNode(Opcode,
DL, MVT::Other, Ops));
1313 LoadNode = cast<LoadSDNode>(Load);
1316 if (!Load.hasOneUse())
1328 bool ChainCheck =
false;
1329 if (Chain == Load.getValue(1)) {
1336 const unsigned int Max = 1024;
1339 if (
Op == Load.getValue(1)) {
1352 if (
Op.getNode() != LoadNode)
1363 MVT::Other, ChainOps);
1380bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(
SDNode *
Node) {
1390 unsigned NewOpc = 0;
1391 bool NegateOperand =
false;
1396 NegateOperand =
true;
1399 if (MemVT == MVT::i32)
1400 NewOpc = SystemZ::ASI;
1401 else if (MemVT == MVT::i64)
1402 NewOpc = SystemZ::AGSI;
1407 NegateOperand =
true;
1410 if (MemVT == MVT::i32)
1411 NewOpc = SystemZ::ALSI;
1412 else if (MemVT == MVT::i64)
1413 NewOpc = SystemZ::ALGSI;
1426 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1429 auto OperandV = OperandC->getAPIntValue();
1431 OperandV = -OperandV;
1432 if (OperandV.getSignificantBits() > 8)
1434 Operand = CurDAG->getTargetConstant(OperandV,
DL, MemVT);
1440 SDValue Ops[] = {
Base, Disp, Operand, InputChain };
1442 CurDAG->getMachineNode(NewOpc,
DL, MVT::i32, MVT::Other, Ops);
1443 CurDAG->setNodeMemRefs(
1448 CurDAG->RemoveDeadNode(
Node);
1452bool SystemZDAGToDAGISel::canUseBlockOperation(
StoreSDNode *Store,
1455 if (
Load->getMemoryVT() !=
Store->getMemoryVT())
1459 if (
Load->isVolatile() ||
Store->isVolatile())
1463 if (
Load->isInvariant() &&
Load->isDereferenceable())
1467 const Value *V1 =
Load->getMemOperand()->getValue();
1474 int64_t End1 =
Load->getSrcValueOffset() +
Size;
1475 int64_t End2 =
Store->getSrcValueOffset() +
Size;
1476 if (V1 == V2 && End1 == End2)
1483bool SystemZDAGToDAGISel::storeLoadCanUseMVC(
SDNode *
N)
const {
1484 auto *
Store = cast<StoreSDNode>(
N);
1485 auto *
Load = cast<LoadSDNode>(
Store->getValue());
1499 return canUseBlockOperation(Store, Load);
1502bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(
SDNode *
N,
1504 auto *StoreA = cast<StoreSDNode>(
N);
1505 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 -
I));
1506 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(
I));
1507 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1508 canUseBlockOperation(StoreA, LoadB);
1511bool SystemZDAGToDAGISel::storeLoadIsAligned(
SDNode *
N)
const {
1513 auto *MemAccess = cast<MemSDNode>(
N);
1514 auto *LdSt = dyn_cast<LSBaseSDNode>(MemAccess);
1515 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1518 assert(MMO &&
"Expected a memory operand.");
1522 if (MemAccess->getAlign().value() < StoreSize ||
1523 (LdSt && !LdSt->getOffset().isUndef()))
1532 if ((PSV->isGOT() || PSV->isConstantPool()))
1538 dyn_cast<GlobalAddressSDNode>(
BasePtr.getOperand(0))) {
1540 if (GA->getOffset() % StoreSize != 0)
1555 if (
auto *L = dyn_cast<LoadSDNode>(
N))
1556 ETy =
L->getExtensionType();
1557 else if (
auto *AL = dyn_cast<AtomicSDNode>(
N))
1558 ETy =
AL->getExtensionType();
1564void SystemZDAGToDAGISel::Select(
SDNode *
Node) {
1566 if (
Node->isMachineOpcode()) {
1568 Node->setNodeId(-1);
1572 unsigned Opcode =
Node->getOpcode();
1576 if (tryRxSBG(
Node, SystemZ::ROSBG))
1582 if (tryRxSBG(
Node, SystemZ::RXSBG))
1589 if (
Node->getValueType(0) == MVT::i64 &&
1591 if (
auto *Op1 = dyn_cast<ConstantSDNode>(
Node->getOperand(1))) {
1592 uint64_t Val = Op1->getZExtValue();
1595 if (Subtarget->hasMiscellaneousExtensions3()) {
1596 unsigned ChildOpcode =
Node->getOperand(0).getOpcode();
1605 auto Op0 =
Node->getOperand(0);
1606 if (
auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->
getOperand(1)))
1607 if (Op0Op1->getZExtValue() == (
uint64_t)-1)
1612 if (Opcode ==
ISD::XOR && Op1->isAllOnes())
1615 splitLargeImmediate(Opcode,
Node,
Node->getOperand(0),
1624 if (tryRxSBG(
Node, SystemZ::RNSBG))
1631 if (tryRISBGZero(
Node))
1636 if (
Node->getValueType(0) == MVT::i128) {
1641 uint64_t Bytes[2] = { 0x0706050403020100ULL, 0x0f0e0d0c0b0a0908ULL };
1645 MVT::v16i8, Ops), 0);
1649 ReplaceNode(
Node, ResNode);
1650 SelectCode(Src.getNode());
1651 SelectCode(ResNode);
1659 if (
Node->getValueType(0) == MVT::i64) {
1667 if (
Node->getValueType(0) == MVT::i128) {
1668 const APInt &Val =
Node->getAsAPIntVal();
1671 loadVectorConstant(VCI,
Node);
1675 SDNode *ResNode = loadPoolVectorConstant(Val, MVT::i128,
SDLoc(
Node));
1676 ReplaceNode(
Node, ResNode);
1687 (Subtarget->hasLoadStoreOnCond2() &&
1688 Node->getValueType(0).isInteger() &&
1689 Node->getValueType(0).getSizeInBits() <= 64 &&
1691 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1693 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1699 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1703 CurDAG->UpdateNodeOperands(
Node, Op1, Op0, CCValid, CCMask, Op4);
1704 if (UpdatedNode !=
Node) {
1706 ReplaceNode(
Node, UpdatedNode);
1714 EVT VT =
Node->getValueType(0);
1716 if (ElemBitSize == 32) {
1717 if (tryGather(
Node, SystemZ::VGEF))
1719 }
else if (ElemBitSize == 64) {
1720 if (tryGather(
Node, SystemZ::VGEG))
1727 auto *BVN = cast<BuildVectorSDNode>(
Node);
1730 loadVectorConstant(VCI,
Node);
1738 if (
Imm.isZero() ||
Imm.isNegZero())
1743 loadVectorConstant(VCI,
Node);
1748 if (tryFoldLoadStoreIntoMemOperand(
Node))
1751 unsigned ElemBitSize =
Store->getValue().getValueSizeInBits();
1752 if (ElemBitSize == 32) {
1753 if (tryScatter(Store, SystemZ::VSCEF))
1755 }
else if (ElemBitSize == 64) {
1756 if (tryScatter(Store, SystemZ::VSCEG))
1763 auto *AtomOp = cast<AtomicSDNode>(
Node);
1767 StoreSDNode *St = cast<StoreSDNode>(CurDAG->getTruncStore(
1768 AtomOp->getChain(),
SDLoc(AtomOp), AtomOp->getVal(),
1769 AtomOp->getBasePtr(), AtomOp->getMemoryVT(), AtomOp->getMemOperand()));
1774 if (AtomOp->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent)
1775 Chain = CurDAG->getMachineNode(SystemZ::Serialize,
SDLoc(AtomOp),
1776 MVT::Other,
SDValue(Chain, 0));
1777 ReplaceNode(
Node, Chain);
1786bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand(
1788 std::vector<SDValue> &OutOps) {
1789 SystemZAddressingMode::AddrForm
Form;
1790 SystemZAddressingMode::DispRange DispRange;
1793 switch(ConstraintID) {
1796 case InlineAsm::ConstraintCode::i:
1797 case InlineAsm::ConstraintCode::Q:
1798 case InlineAsm::ConstraintCode::ZQ:
1800 Form = SystemZAddressingMode::FormBD;
1801 DispRange = SystemZAddressingMode::Disp12Only;
1803 case InlineAsm::ConstraintCode::R:
1804 case InlineAsm::ConstraintCode::ZR:
1806 Form = SystemZAddressingMode::FormBDXNormal;
1807 DispRange = SystemZAddressingMode::Disp12Only;
1809 case InlineAsm::ConstraintCode::S:
1810 case InlineAsm::ConstraintCode::ZS:
1812 Form = SystemZAddressingMode::FormBD;
1813 DispRange = SystemZAddressingMode::Disp20Only;
1815 case InlineAsm::ConstraintCode::T:
1816 case InlineAsm::ConstraintCode::m:
1817 case InlineAsm::ConstraintCode::o:
1818 case InlineAsm::ConstraintCode::p:
1819 case InlineAsm::ConstraintCode::ZT:
1824 Form = SystemZAddressingMode::FormBDXNormal;
1825 DispRange = SystemZAddressingMode::Disp20Only;
1831 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1833 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(),
DL, MVT::i32);
1840 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1848 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1853 OutOps.push_back(
Base);
1854 OutOps.push_back(Disp);
1855 OutOps.push_back(
Index);
1870 if (!
N.hasOneUse() || !
U->hasOneUse())
1877 SDNode *CCUser = *
U->use_begin();
1878 SDNode *CCRegUser =
nullptr;
1880 cast<RegisterSDNode>(CCUser->
getOperand(1))->getReg() == SystemZ::CC) {
1881 for (
auto *U : CCUser->
uses()) {
1882 if (CCRegUser ==
nullptr)
1884 else if (CCRegUser != U)
1888 if (CCRegUser ==
nullptr)
1901 if (!IsLegalToFold(
N, U, CCRegUser, OptLevel,
false))
1911struct IPMConversion {
1912 IPMConversion(
unsigned xorValue, int64_t addValue,
unsigned bit)
1913 : XORValue(xorValue), AddValue(addValue),
Bit(bit) {}
1989 auto *TrueOp = dyn_cast<ConstantSDNode>(
Node->getOperand(0));
1990 auto *FalseOp = dyn_cast<ConstantSDNode>(
Node->getOperand(1));
1991 if (!TrueOp || !FalseOp)
1993 if (FalseOp->getZExtValue() != 0)
1995 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
1998 auto *CCValidOp = dyn_cast<ConstantSDNode>(
Node->getOperand(2));
1999 auto *CCMaskOp = dyn_cast<ConstantSDNode>(
Node->getOperand(3));
2000 if (!CCValidOp || !CCMaskOp)
2002 int CCValid = CCValidOp->getZExtValue();
2003 int CCMask = CCMaskOp->getZExtValue();
2012 CurDAG->getConstant(
IPM.XORValue,
DL, MVT::i32));
2016 CurDAG->getConstant(
IPM.AddValue,
DL, MVT::i32));
2018 EVT VT =
Node->getValueType(0);
2019 if (VT == MVT::i32 &&
IPM.Bit == 31) {
2021 Result = CurDAG->getNode(ShiftOp,
DL, MVT::i32, Result,
2022 CurDAG->getConstant(
IPM.Bit,
DL, MVT::i32));
2027 if (TrueOp->getSExtValue() == 1) {
2030 CurDAG->getConstant(
IPM.Bit,
DL, MVT::i32));
2032 CurDAG->getConstant(1,
DL, VT));
2038 CurDAG->getConstant(ShlAmt,
DL, MVT::i32));
2040 CurDAG->getConstant(SraAmt,
DL, MVT::i32));
2047void SystemZDAGToDAGISel::PreprocessISelDAG() {
2050 if (Subtarget->hasLoadStoreOnCond2())
2053 bool MadeChange =
false;
2056 E = CurDAG->allnodes_end();
2063 switch (
N->getOpcode()) {
2066 Res = expandSelectBoolean(
N);
2071 LLVM_DEBUG(
dbgs() <<
"SystemZ DAG preprocessing replacing:\nOld: ");
2077 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
2083 CurDAG->RemoveDeadNodes();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
const HexagonInstrInfo * TII
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask)
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
static uint64_t allOnes(unsigned int Count)
DEMANGLE_DUMP_METHOD void dump() const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
Module * getParent()
Get the module that this global value is contained inside of...
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
bool isAtomic() const
Returns true if this operation has an atomic ordering requirement of unordered or higher,...
@ MOLoad
The memory access reads data.
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
An SDNode that represents everything that will be needed to construct a MachineInstr.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
bool isNonTemporal() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Special value supplied for machine level alias analysis.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
static int getUninvalidatedNodeId(SDNode *N)
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SystemZInstrInfo * getInstrInfo() const override
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
static IntegerType * getInt128Ty(LLVMContext &C)
LLVM Value Representation.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ BSWAP
Byte Swap and Counting operators.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SHL
Shift and rotation operations.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isPCREL(unsigned Opcode)
static bool isImmHF(uint64_t Val)
static bool isImmLF(uint64_t Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CodeGenOptLevel
Code generation optimization level.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Or
Bitwise or logical OR of integers.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)