23#define DEBUG_TYPE "systemz-isel"
24#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
28struct SystemZAddressingMode {
63 bool IncludesDynAlloc;
65 SystemZAddressingMode(AddrForm form, DispRange dr)
66 :
Form(form), DR(dr), Disp(0), IncludesDynAlloc(
false) {}
69 bool hasIndexField() {
return Form != FormBD; }
72 bool isDynAlloc() {
return Form == FormBDXDynAlloc; }
75 errs() <<
"SystemZAddressingMode " <<
this <<
'\n';
79 Base.getNode()->dump(DAG);
83 if (hasIndexField()) {
86 Index.getNode()->dump(DAG);
91 errs() <<
" Disp " << Disp;
93 errs() <<
" + ADJDYNALLOC";
119struct RxSBGOperands {
121 : Opcode(
Op), BitSize(
N.getValueSizeInBits()),
122 Mask(
allOnes(BitSize)), Input(
N), Start(64 - BitSize),
End(63),
139 return CurDAG->getTargetConstant(Imm,
SDLoc(
Node),
Node->getValueType(0));
152 bool expandAddress(SystemZAddressingMode &AM,
bool IsBase)
const;
155 bool selectAddress(
SDValue N, SystemZAddressingMode &AM)
const;
158 void getAddressOperands(
const SystemZAddressingMode &AM,
EVT VT,
160 void getAddressOperands(
const SystemZAddressingMode &AM,
EVT VT,
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR,
SDValue Addr,
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR,
SDValue Addr,
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm
Form,
193 return selectBDAddr(SystemZAddressingMode::Disp12Only,
Addr,
Base, Disp);
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair,
Addr,
Base, Disp);
199 return selectBDAddr(SystemZAddressingMode::Disp20Only,
Addr,
Base, Disp);
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair,
Addr,
Base, Disp);
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair,
Addr,
Base, Disp);
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair,
Addr,
Base, Disp);
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
277 bool refineRxSBGMask(RxSBGOperands &RxSBG,
uint64_t Mask)
const;
281 bool expandRxSBG(RxSBGOperands &RxSBG)
const;
295 bool tryRxSBG(
SDNode *
N,
unsigned Opcode);
313 bool tryGather(
SDNode *
N,
unsigned Opcode);
316 bool tryScatter(
StoreSDNode *Store,
unsigned Opcode);
321 bool tryFoldLoadStoreIntoMemOperand(
SDNode *
Node);
336 bool storeLoadCanUseMVC(
SDNode *
N)
const;
341 bool storeLoadCanUseBlockBinary(
SDNode *
N,
unsigned I)
const;
345 bool storeLoadIsAligned(
SDNode *
N)
const;
356 bool shouldSelectForReassoc(
SDNode *
N)
const;
361 SystemZDAGToDAGISel() =
delete;
368 if (
F.getFnAttribute(
"fentry-call").getValueAsString() !=
"true") {
369 if (
F.hasFnAttribute(
"mnop-mcount"))
371 if (
F.hasFnAttribute(
"mrecord-mcount"))
383 std::vector<SDValue> &OutOps)
override;
388 #include "SystemZGenDAGISel.inc"
392char SystemZDAGToDAGISel::ID = 0;
398 return new SystemZDAGToDAGISel(
TM, OptLevel);
404static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
406 case SystemZAddressingMode::Disp12Only:
407 return isUInt<12>(Val);
409 case SystemZAddressingMode::Disp12Pair:
410 case SystemZAddressingMode::Disp20Only:
411 case SystemZAddressingMode::Disp20Pair:
412 return isInt<20>(Val);
414 case SystemZAddressingMode::Disp20Only128:
415 return isInt<20>(Val) && isInt<20>(Val + 8);
435 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
437 AM.IncludesDynAlloc =
true;
447 if (AM.hasIndexField() && !AM.Index.getNode()) {
457static bool expandDisp(SystemZAddressingMode &AM,
bool IsBase,
460 int64_t TestDisp = AM.Disp + Op1;
472bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
475 unsigned Opcode =
N.getOpcode();
477 if (Opcode ==
ISD::TRUNCATE &&
N.getOperand(0).getValueSizeInBits() <= 64) {
479 Opcode =
N.getOpcode();
481 if (Opcode ==
ISD::ADD || CurDAG->isBaseWithConstantOffset(
N)) {
495 cast<ConstantSDNode>(Op0)->getSExtValue());
498 cast<ConstantSDNode>(Op1)->getSExtValue());
508 cast<GlobalAddressSDNode>(Anchor)->getOffset());
516static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
519 case SystemZAddressingMode::Disp12Only:
520 case SystemZAddressingMode::Disp20Only:
521 case SystemZAddressingMode::Disp20Only128:
524 case SystemZAddressingMode::Disp12Pair:
526 return isUInt<12>(Val);
528 case SystemZAddressingMode::Disp20Pair:
530 return !isUInt<12>(Val);
554 if (isUInt<12>(Disp))
559 if (!isInt<16>(Disp))
568 if (
Index->hasOneUse())
573 unsigned IndexOpcode =
Index->getOpcode();
581 if (
Base->hasOneUse())
588bool SystemZDAGToDAGISel::selectAddress(
SDValue Addr,
589 SystemZAddressingMode &AM)
const {
597 cast<ConstantSDNode>(
Addr)->getSExtValue()))
605 while (expandAddress(AM,
true) ||
606 (AM.Index.getNode() && expandAddress(AM,
false)))
610 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
611 !
shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
619 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
632 if (
N->getNodeId() == -1 ||
645void SystemZDAGToDAGISel::getAddressOperands(
const SystemZAddressingMode &AM,
651 Base = CurDAG->getRegister(0, VT);
655 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
656 }
else if (
Base.getValueType() != VT) {
658 assert(VT == MVT::i32 &&
Base.getValueType() == MVT::i64 &&
659 "Unexpected truncation");
667 Disp = CurDAG->getTargetConstant(AM.Disp,
SDLoc(
Base), VT);
670void SystemZDAGToDAGISel::getAddressOperands(
const SystemZAddressingMode &AM,
674 getAddressOperands(AM, VT,
Base, Disp);
677 if (!
Index.getNode())
679 Index = CurDAG->getRegister(0, VT);
682bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
685 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
686 if (!selectAddress(
Addr, AM))
689 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp);
693bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
696 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
697 if (!selectAddress(
Addr, AM) || AM.Index.getNode())
700 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp);
704bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm
Form,
705 SystemZAddressingMode::DispRange DR,
708 SystemZAddressingMode AM(
Form, DR);
709 if (!selectAddress(
Addr, AM))
712 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp,
Index);
721 if (selectBDXAddr12Only(
Addr, Regs[0], Disp, Regs[1]) &&
722 Regs[0].getNode() && Regs[1].getNode()) {
723 for (
unsigned int I = 0;
I < 2; ++
I) {
731 Index.getOperand(1) == Elem) {
740bool SystemZDAGToDAGISel::detectOrAndInsertion(
SDValue &
Op,
748 auto *MaskNode = dyn_cast<ConstantSDNode>(
Op.getOperand(1).getNode());
753 uint64_t AndMask = MaskNode->getZExtValue();
754 if (InsertMask & AndMask)
760 if (Used != (AndMask | InsertMask)) {
761 KnownBits Known = CurDAG->computeKnownBits(
Op.getOperand(0));
766 Op =
Op.getOperand(0);
770bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
773 if (RxSBG.Rotate != 0)
774 Mask = (
Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
776 if (
TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
786 if (RxSBG.Rotate != 0)
787 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
788 return (Mask & RxSBG.Mask) != 0;
791bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG)
const {
793 unsigned Opcode =
N.getOpcode();
796 if (RxSBG.Opcode == SystemZ::RNSBG)
798 if (
N.getOperand(0).getValueSizeInBits() > 64)
800 uint64_t BitSize =
N.getValueSizeInBits();
802 if (!refineRxSBGMask(RxSBG, Mask))
804 RxSBG.Input =
N.getOperand(0);
808 if (RxSBG.Opcode == SystemZ::RNSBG)
811 auto *MaskNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
817 if (!refineRxSBGMask(RxSBG, Mask)) {
821 KnownBits Known = CurDAG->computeKnownBits(Input);
823 if (!refineRxSBGMask(RxSBG, Mask))
831 if (RxSBG.Opcode != SystemZ::RNSBG)
834 auto *MaskNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
840 if (!refineRxSBGMask(RxSBG, Mask)) {
844 KnownBits Known = CurDAG->computeKnownBits(Input);
845 Mask &= ~Known.One.getZExtValue();
846 if (!refineRxSBGMask(RxSBG, Mask))
855 if (RxSBG.BitSize != 64 ||
N.getValueType() != MVT::i64)
857 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
861 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
862 RxSBG.Input =
N.getOperand(0);
868 RxSBG.Input =
N.getOperand(0);
872 if (RxSBG.Opcode != SystemZ::RNSBG) {
874 unsigned InnerBitSize =
N.getOperand(0).getValueSizeInBits();
875 if (!refineRxSBGMask(RxSBG,
allOnes(InnerBitSize)))
878 RxSBG.Input =
N.getOperand(0);
886 unsigned BitSize =
N.getValueSizeInBits();
887 unsigned InnerBitSize =
N.getOperand(0).getValueSizeInBits();
891 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
892 RxSBG.Rotate += (BitSize - InnerBitSize);
897 RxSBG.Input =
N.getOperand(0);
902 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
906 uint64_t Count = CountNode->getZExtValue();
907 unsigned BitSize =
N.getValueSizeInBits();
908 if (Count < 1 || Count >= BitSize)
911 if (RxSBG.Opcode == SystemZ::RNSBG) {
918 if (!refineRxSBGMask(RxSBG,
allOnes(BitSize - Count) << Count))
922 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
923 RxSBG.Input =
N.getOperand(0);
929 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
933 uint64_t Count = CountNode->getZExtValue();
934 unsigned BitSize =
N.getValueSizeInBits();
935 if (Count < 1 || Count >= BitSize)
938 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode ==
ISD::SRA) {
946 if (!refineRxSBGMask(RxSBG,
allOnes(BitSize - Count)))
950 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
951 RxSBG.Input =
N.getOperand(0);
960 SDNode *
N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, VT);
966 if (
N.getValueType() == MVT::i32 && VT == MVT::i64)
967 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
968 DL, VT, getUNDEF(
DL, MVT::i64),
N);
969 if (
N.getValueType() == MVT::i64 && VT == MVT::i32)
970 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32,
DL, VT,
N);
971 assert(
N.getValueType() == VT &&
"Unexpected value types");
975bool SystemZDAGToDAGISel::tryRISBGZero(
SDNode *
N) {
977 EVT VT =
N->getValueType(0);
980 RxSBGOperands RISBG(SystemZ::RISBG,
SDValue(
N, 0));
982 while (expandRxSBG(RISBG))
989 if (Count == 0 || isa<ConstantSDNode>(RISBG.Input))
994 if (Count == 1 &&
N->getOpcode() !=
ISD::AND)
1000 if (RISBG.Rotate == 0) {
1001 bool PreferAnd =
false;
1007 else if (RISBG.Mask == 0xff ||
1008 RISBG.Mask == 0xffff ||
1009 RISBG.Mask == 0x7fffffff ||
1015 else if (
auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
1016 if (
Load->getMemoryVT() == MVT::i32 &&
1019 RISBG.Mask == 0xffffff00 &&
1020 Subtarget->hasLoadAndZeroRightmostByte())
1030 if (
N !=
New.getNode()) {
1033 ReplaceNode(
N,
New.getNode());
1037 if (!
N->isMachineOpcode())
1043 unsigned Opcode = SystemZ::RISBG;
1045 if (Subtarget->hasMiscellaneousExtensions())
1046 Opcode = SystemZ::RISBGN;
1047 EVT OpcodeVT = MVT::i64;
1048 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1053 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1054 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1055 ((RISBG.End + RISBG.Rotate) & 63) >=
1056 ((RISBG.Start + RISBG.Rotate) & 63)) {
1057 Opcode = SystemZ::RISBMux;
1058 OpcodeVT = MVT::i32;
1063 getUNDEF(
DL, OpcodeVT),
1064 convertTo(
DL, OpcodeVT, RISBG.Input),
1065 CurDAG->getTargetConstant(RISBG.Start,
DL, MVT::i32),
1066 CurDAG->getTargetConstant(RISBG.End | 128,
DL, MVT::i32),
1067 CurDAG->getTargetConstant(RISBG.Rotate,
DL, MVT::i32)
1070 DL, VT,
SDValue(CurDAG->getMachineNode(Opcode,
DL, OpcodeVT, Ops), 0));
1071 ReplaceNode(
N,
New.getNode());
1075bool SystemZDAGToDAGISel::tryRxSBG(
SDNode *
N,
unsigned Opcode) {
1077 EVT VT =
N->getValueType(0);
1082 RxSBGOperands RxSBG[] = {
1083 RxSBGOperands(Opcode,
N->getOperand(0)),
1084 RxSBGOperands(Opcode,
N->getOperand(1))
1086 unsigned Count[] = { 0, 0 };
1087 for (
unsigned I = 0;
I < 2; ++
I)
1088 while (RxSBG[
I].Input->
hasOneUse() && expandRxSBG(RxSBG[
I]))
1100 if (Count[0] == 0 && Count[1] == 0)
1104 unsigned I = Count[0] > Count[1] ? 0 : 1;
1108 if (Opcode == SystemZ::ROSBG && (RxSBG[
I].Mask & 0xff) == 0)
1109 if (
auto *Load = dyn_cast<LoadSDNode>(Op0.
getNode()))
1110 if (
Load->getMemoryVT() == MVT::i8)
1115 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[
I].Mask)) {
1116 Opcode = SystemZ::RISBG;
1118 if (Subtarget->hasMiscellaneousExtensions())
1119 Opcode = SystemZ::RISBGN;
1123 convertTo(
DL, MVT::i64, Op0),
1124 convertTo(
DL, MVT::i64, RxSBG[
I].Input),
1125 CurDAG->getTargetConstant(RxSBG[
I].Start,
DL, MVT::i32),
1126 CurDAG->getTargetConstant(RxSBG[
I].
End,
DL, MVT::i32),
1127 CurDAG->getTargetConstant(RxSBG[
I].Rotate,
DL, MVT::i32)
1130 DL, VT,
SDValue(CurDAG->getMachineNode(Opcode,
DL, MVT::i64, Ops), 0));
1131 ReplaceNode(
N,
New.getNode());
1135void SystemZDAGToDAGISel::splitLargeImmediate(
unsigned Opcode,
SDNode *
Node,
1138 EVT VT =
Node->getValueType(0);
1158 SelectCode(
Upper.getNode());
1159 Upper = Handle.getValue();
1165 ReplaceNode(
Node,
Or.getNode());
1167 SelectCode(
Or.getNode());
1170void SystemZDAGToDAGISel::loadVectorConstant(
1177 EVT VT =
Node->getValueType(0);
1180 for (
unsigned OpVal : VCI.
OpVals)
1181 Ops.
push_back(CurDAG->getTargetConstant(OpVal,
DL, MVT::i32));
1185 ReplaceNode(
Node,
Op.getNode());
1189 SelectCode(BitCast.
getNode());
1191 unsigned SubRegIdx =
1192 (VT.
getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1194 Node, CurDAG->getTargetExtractSubreg(SubRegIdx,
DL, VT,
Op).getNode());
1196 SelectCode(
Op.getNode());
1203 SDValue CP = CurDAG->getTargetConstantPool(
1205 TLI->getPointerTy(CurDAG->getDataLayout()));
1207 EVT PtrVT =
CP.getValueType();
1209 SDValue(CurDAG->getMachineNode(SystemZ::LARL,
DL, PtrVT, CP), 0),
1210 CurDAG->getTargetConstant(0,
DL, PtrVT),
1211 CurDAG->getRegister(0, PtrVT),
1212 CurDAG->getEntryNode()
1214 ResNode = CurDAG->getMachineNode(SystemZ::VL,
DL, VT, MVT::Other, Ops);
1224 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
1228bool SystemZDAGToDAGISel::tryGather(
SDNode *
N,
unsigned Opcode) {
1230 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1234 unsigned Elem = ElemN->getZExtValue();
1235 EVT VT =
N->getValueType(0);
1239 auto *
Load = dyn_cast<LoadSDNode>(
N->getOperand(1));
1240 if (!Load || !
Load->hasNUsesOfValue(1, 0))
1242 if (
Load->getMemoryVT().getSizeInBits() !=
1243 Load->getValueType(0).getSizeInBits())
1247 if (!selectBDVAddr12Only(
Load->getBasePtr(), ElemV,
Base, Disp,
Index) ||
1254 CurDAG->getTargetConstant(Elem,
DL, MVT::i32),
Load->getChain()
1256 SDNode *Res = CurDAG->getMachineNode(Opcode,
DL, VT, MVT::Other, Ops);
1258 ReplaceNode(
N, Res);
1262bool SystemZDAGToDAGISel::tryScatter(
StoreSDNode *Store,
unsigned Opcode) {
1266 if (
Store->getMemoryVT().getSizeInBits() !=
Value.getValueSizeInBits())
1270 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1276 unsigned Elem = ElemN->getZExtValue();
1281 if (!selectBDVAddr12Only(
Store->getBasePtr(), ElemV,
Base, Disp,
Index) ||
1287 Vec,
Base, Disp,
Index, CurDAG->getTargetConstant(Elem,
DL, MVT::i32),
1290 ReplaceNode(Store, CurDAG->getMachineNode(Opcode,
DL, MVT::Other, Ops));
1318 LoadNode = cast<LoadSDNode>(Load);
1321 if (!Load.hasOneUse())
1333 bool ChainCheck =
false;
1334 if (Chain == Load.getValue(1)) {
1341 const unsigned int Max = 1024;
1344 if (
Op == Load.getValue(1)) {
1357 if (
Op.getNode() != LoadNode)
1368 MVT::Other, ChainOps);
1385bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(
SDNode *
Node) {
1395 unsigned NewOpc = 0;
1396 bool NegateOperand =
false;
1401 NegateOperand =
true;
1404 if (MemVT == MVT::i32)
1405 NewOpc = SystemZ::ASI;
1406 else if (MemVT == MVT::i64)
1407 NewOpc = SystemZ::AGSI;
1412 NegateOperand =
true;
1415 if (MemVT == MVT::i32)
1416 NewOpc = SystemZ::ALSI;
1417 else if (MemVT == MVT::i64)
1418 NewOpc = SystemZ::ALGSI;
1431 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1434 auto OperandV = OperandC->getAPIntValue();
1436 OperandV = -OperandV;
1437 if (OperandV.getSignificantBits() > 8)
1439 Operand = CurDAG->getTargetConstant(OperandV,
DL, MemVT);
1445 SDValue Ops[] = {
Base, Disp, Operand, InputChain };
1447 CurDAG->getMachineNode(NewOpc,
DL, MVT::i32, MVT::Other, Ops);
1448 CurDAG->setNodeMemRefs(
1453 CurDAG->RemoveDeadNode(
Node);
1457bool SystemZDAGToDAGISel::canUseBlockOperation(
StoreSDNode *Store,
1460 if (
Load->getMemoryVT() !=
Store->getMemoryVT())
1464 if (
Load->isVolatile() ||
Store->isVolatile())
1468 if (
Load->isInvariant() &&
Load->isDereferenceable())
1472 const Value *V1 =
Load->getMemOperand()->getValue();
1479 int64_t End1 =
Load->getSrcValueOffset() +
Size;
1480 int64_t End2 =
Store->getSrcValueOffset() +
Size;
1481 if (V1 == V2 && End1 == End2)
1488bool SystemZDAGToDAGISel::storeLoadCanUseMVC(
SDNode *
N)
const {
1489 auto *
Store = cast<StoreSDNode>(
N);
1490 auto *
Load = cast<LoadSDNode>(
Store->getValue());
1504 return canUseBlockOperation(Store, Load);
1507bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(
SDNode *
N,
1509 auto *StoreA = cast<StoreSDNode>(
N);
1510 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 -
I));
1511 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(
I));
1512 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1513 canUseBlockOperation(StoreA, LoadB);
1516bool SystemZDAGToDAGISel::storeLoadIsAligned(
SDNode *
N)
const {
1518 auto *MemAccess = cast<MemSDNode>(
N);
1519 auto *LdSt = dyn_cast<LSBaseSDNode>(MemAccess);
1520 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1523 assert(MMO &&
"Expected a memory operand.");
1527 if (MemAccess->getAlign().value() < StoreSize ||
1528 (LdSt && !LdSt->getOffset().isUndef()))
1537 if ((PSV->isGOT() || PSV->isConstantPool()))
1543 dyn_cast<GlobalAddressSDNode>(
BasePtr.getOperand(0))) {
1545 if (GA->getOffset() % StoreSize != 0)
1560 if (
auto *L = dyn_cast<LoadSDNode>(
N))
1561 ETy =
L->getExtensionType();
1562 else if (
auto *AL = dyn_cast<AtomicSDNode>(
N))
1563 ETy =
AL->getExtensionType();
1569void SystemZDAGToDAGISel::Select(
SDNode *
Node) {
1571 if (
Node->isMachineOpcode()) {
1573 Node->setNodeId(-1);
1577 unsigned Opcode =
Node->getOpcode();
1581 if (tryRxSBG(
Node, SystemZ::ROSBG))
1587 if (tryRxSBG(
Node, SystemZ::RXSBG))
1594 if (
Node->getValueType(0) == MVT::i64 &&
1596 if (
auto *Op1 = dyn_cast<ConstantSDNode>(
Node->getOperand(1))) {
1597 uint64_t Val = Op1->getZExtValue();
1600 if (Subtarget->hasMiscellaneousExtensions3()) {
1601 unsigned ChildOpcode =
Node->getOperand(0).getOpcode();
1610 auto Op0 =
Node->getOperand(0);
1611 if (
auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->
getOperand(1)))
1612 if (Op0Op1->getZExtValue() == (
uint64_t)-1)
1617 if (Opcode ==
ISD::XOR && Op1->isAllOnes())
1620 splitLargeImmediate(Opcode,
Node,
Node->getOperand(0),
1629 if (tryRxSBG(
Node, SystemZ::RNSBG))
1636 if (tryRISBGZero(
Node))
1641 if (
Node->getValueType(0) == MVT::i128) {
1646 uint64_t Bytes[2] = { 0x0706050403020100ULL, 0x0f0e0d0c0b0a0908ULL };
1650 MVT::v16i8, Ops), 0);
1654 ReplaceNode(
Node, ResNode);
1655 SelectCode(Src.getNode());
1656 SelectCode(ResNode);
1664 if (
Node->getValueType(0) == MVT::i64) {
1672 if (
Node->getValueType(0) == MVT::i128) {
1673 const APInt &Val =
Node->getAsAPIntVal();
1676 loadVectorConstant(VCI,
Node);
1680 SDNode *ResNode = loadPoolVectorConstant(Val, MVT::i128,
SDLoc(
Node));
1681 ReplaceNode(
Node, ResNode);
1692 (Subtarget->hasLoadStoreOnCond2() &&
1693 Node->getValueType(0).isInteger() &&
1694 Node->getValueType(0).getSizeInBits() <= 64 &&
1696 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1698 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1704 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1708 CurDAG->UpdateNodeOperands(
Node, Op1, Op0, CCValid, CCMask, Op4);
1709 if (UpdatedNode !=
Node) {
1711 ReplaceNode(
Node, UpdatedNode);
1719 EVT VT =
Node->getValueType(0);
1721 if (ElemBitSize == 32) {
1722 if (tryGather(
Node, SystemZ::VGEF))
1724 }
else if (ElemBitSize == 64) {
1725 if (tryGather(
Node, SystemZ::VGEG))
1732 auto *BVN = cast<BuildVectorSDNode>(
Node);
1735 loadVectorConstant(VCI,
Node);
1743 if (
Imm.isZero() ||
Imm.isNegZero())
1748 loadVectorConstant(VCI,
Node);
1753 if (tryFoldLoadStoreIntoMemOperand(
Node))
1756 unsigned ElemBitSize =
Store->getValue().getValueSizeInBits();
1757 if (ElemBitSize == 32) {
1758 if (tryScatter(Store, SystemZ::VSCEF))
1760 }
else if (ElemBitSize == 64) {
1761 if (tryScatter(Store, SystemZ::VSCEG))
1768 auto *AtomOp = cast<AtomicSDNode>(
Node);
1772 StoreSDNode *St = cast<StoreSDNode>(CurDAG->getTruncStore(
1773 AtomOp->getChain(),
SDLoc(AtomOp), AtomOp->getVal(),
1774 AtomOp->getBasePtr(), AtomOp->getMemoryVT(), AtomOp->getMemOperand()));
1779 if (AtomOp->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent)
1780 Chain = CurDAG->getMachineNode(SystemZ::Serialize,
SDLoc(AtomOp),
1781 MVT::Other,
SDValue(Chain, 0));
1782 ReplaceNode(
Node, Chain);
1791bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand(
1793 std::vector<SDValue> &OutOps) {
1794 SystemZAddressingMode::AddrForm
Form;
1795 SystemZAddressingMode::DispRange DispRange;
1798 switch(ConstraintID) {
1801 case InlineAsm::ConstraintCode::i:
1802 case InlineAsm::ConstraintCode::Q:
1803 case InlineAsm::ConstraintCode::ZQ:
1805 Form = SystemZAddressingMode::FormBD;
1806 DispRange = SystemZAddressingMode::Disp12Only;
1808 case InlineAsm::ConstraintCode::R:
1809 case InlineAsm::ConstraintCode::ZR:
1811 Form = SystemZAddressingMode::FormBDXNormal;
1812 DispRange = SystemZAddressingMode::Disp12Only;
1814 case InlineAsm::ConstraintCode::S:
1815 case InlineAsm::ConstraintCode::ZS:
1817 Form = SystemZAddressingMode::FormBD;
1818 DispRange = SystemZAddressingMode::Disp20Only;
1820 case InlineAsm::ConstraintCode::T:
1821 case InlineAsm::ConstraintCode::m:
1822 case InlineAsm::ConstraintCode::o:
1823 case InlineAsm::ConstraintCode::p:
1824 case InlineAsm::ConstraintCode::ZT:
1829 Form = SystemZAddressingMode::FormBDXNormal;
1830 DispRange = SystemZAddressingMode::Disp20Only;
1836 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1838 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(),
DL, MVT::i32);
1845 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1853 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1858 OutOps.push_back(
Base);
1859 OutOps.push_back(Disp);
1860 OutOps.push_back(
Index);
1875 if (!
N.hasOneUse() || !
U->hasOneUse())
1882 SDNode *CCUser = *
U->use_begin();
1883 SDNode *CCRegUser =
nullptr;
1885 cast<RegisterSDNode>(CCUser->
getOperand(1))->getReg() == SystemZ::CC) {
1886 for (
auto *U : CCUser->
uses()) {
1887 if (CCRegUser ==
nullptr)
1889 else if (CCRegUser != U)
1893 if (CCRegUser ==
nullptr)
1906 if (!IsLegalToFold(
N, U, CCRegUser, OptLevel,
false))
1916struct IPMConversion {
1917 IPMConversion(
unsigned xorValue, int64_t addValue,
unsigned bit)
1918 : XORValue(xorValue), AddValue(addValue),
Bit(bit) {}
1994 auto *TrueOp = dyn_cast<ConstantSDNode>(
Node->getOperand(0));
1995 auto *FalseOp = dyn_cast<ConstantSDNode>(
Node->getOperand(1));
1996 if (!TrueOp || !FalseOp)
1998 if (FalseOp->getZExtValue() != 0)
2000 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
2003 auto *CCValidOp = dyn_cast<ConstantSDNode>(
Node->getOperand(2));
2004 auto *CCMaskOp = dyn_cast<ConstantSDNode>(
Node->getOperand(3));
2005 if (!CCValidOp || !CCMaskOp)
2007 int CCValid = CCValidOp->getZExtValue();
2008 int CCMask = CCMaskOp->getZExtValue();
2017 CurDAG->getConstant(
IPM.XORValue,
DL, MVT::i32));
2021 CurDAG->getConstant(
IPM.AddValue,
DL, MVT::i32));
2023 EVT VT =
Node->getValueType(0);
2024 if (VT == MVT::i32 &&
IPM.Bit == 31) {
2026 Result = CurDAG->getNode(ShiftOp,
DL, MVT::i32, Result,
2027 CurDAG->getConstant(
IPM.Bit,
DL, MVT::i32));
2032 if (TrueOp->getSExtValue() == 1) {
2035 CurDAG->getConstant(
IPM.Bit,
DL, MVT::i32));
2037 CurDAG->getConstant(1,
DL, VT));
2043 CurDAG->getConstant(ShlAmt,
DL, MVT::i32));
2045 CurDAG->getConstant(SraAmt,
DL, MVT::i32));
2052bool SystemZDAGToDAGISel::shouldSelectForReassoc(
SDNode *
N)
const {
2053 EVT VT =
N->getValueType(0);
2055 return N->getFlags().hasAllowReassociation() &&
2056 N->getFlags().hasNoSignedZeros() && Subtarget->hasVector() &&
2057 (VT != MVT::f32 || Subtarget->hasVectorEnhancements1()) &&
2058 !
N->isStrictFPOpcode();
2061void SystemZDAGToDAGISel::PreprocessISelDAG() {
2064 if (Subtarget->hasLoadStoreOnCond2())
2067 bool MadeChange =
false;
2070 E = CurDAG->allnodes_end();
2077 switch (
N->getOpcode()) {
2080 Res = expandSelectBoolean(
N);
2085 LLVM_DEBUG(
dbgs() <<
"SystemZ DAG preprocessing replacing:\nOld: ");
2091 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
2097 CurDAG->RemoveDeadNodes();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
const HexagonInstrInfo * TII
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask)
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
static uint64_t allOnes(unsigned int Count)
DEMANGLE_DUMP_METHOD void dump() const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
Module * getParent()
Get the module that this global value is contained inside of...
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
bool isAtomic() const
Returns true if this operation has an atomic ordering requirement of unordered or higher,...
@ MOLoad
The memory access reads data.
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
An SDNode that represents everything that will be needed to construct a MachineInstr.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
bool isNonTemporal() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Special value supplied for machine level alias analysis.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
static int getUninvalidatedNodeId(SDNode *N)
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SystemZInstrInfo * getInstrInfo() const override
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
static IntegerType * getInt128Ty(LLVMContext &C)
LLVM Value Representation.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ BSWAP
Byte Swap and Counting operators.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SHL
Shift and rotation operations.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isPCREL(unsigned Opcode)
static bool isImmHF(uint64_t Val)
static bool isImmLF(uint64_t Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CodeGenOptLevel
Code generation optimization level.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Or
Bitwise or logical OR of integers.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)