23#define DEBUG_TYPE "systemz-isel"
24#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
28struct SystemZAddressingMode {
63 bool IncludesDynAlloc;
65 SystemZAddressingMode(AddrForm form, DispRange dr)
66 :
Form(form), DR(dr), Disp(0), IncludesDynAlloc(
false) {}
69 bool hasIndexField() {
return Form != FormBD; }
72 bool isDynAlloc() {
return Form == FormBDXDynAlloc; }
75 errs() <<
"SystemZAddressingMode " <<
this <<
'\n';
79 Base.getNode()->dump(DAG);
83 if (hasIndexField()) {
86 Index.getNode()->dump(DAG);
91 errs() <<
" Disp " << Disp;
93 errs() <<
" + ADJDYNALLOC";
119struct RxSBGOperands {
121 : Opcode(
Op), BitSize(
N.getValueSizeInBits()),
122 Mask(
allOnes(BitSize)), Input(
N), Start(64 - BitSize),
End(63),
139 return CurDAG->getTargetConstant(Imm,
SDLoc(
Node),
Node->getValueType(0));
152 bool expandAddress(SystemZAddressingMode &AM,
bool IsBase)
const;
155 bool selectAddress(
SDValue N, SystemZAddressingMode &AM)
const;
158 void getAddressOperands(
const SystemZAddressingMode &AM,
EVT VT,
160 void getAddressOperands(
const SystemZAddressingMode &AM,
EVT VT,
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR,
SDValue Addr,
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR,
SDValue Addr,
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm
Form,
193 return selectBDAddr(SystemZAddressingMode::Disp12Only,
Addr,
Base, Disp);
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair,
Addr,
Base, Disp);
199 return selectBDAddr(SystemZAddressingMode::Disp20Only,
Addr,
Base, Disp);
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair,
Addr,
Base, Disp);
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair,
Addr,
Base, Disp);
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair,
Addr,
Base, Disp);
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
277 bool refineRxSBGMask(RxSBGOperands &RxSBG,
uint64_t Mask)
const;
281 bool expandRxSBG(RxSBGOperands &RxSBG)
const;
295 bool tryRxSBG(
SDNode *
N,
unsigned Opcode);
313 bool tryGather(
SDNode *
N,
unsigned Opcode);
316 bool tryScatter(
StoreSDNode *Store,
unsigned Opcode);
321 bool tryFoldLoadStoreIntoMemOperand(
SDNode *
Node);
336 bool storeLoadCanUseMVC(
SDNode *
N)
const;
341 bool storeLoadCanUseBlockBinary(
SDNode *
N,
unsigned I)
const;
345 bool storeLoadIsAligned(
SDNode *
N)
const;
356 bool shouldSelectForReassoc(
SDNode *
N)
const;
359 SystemZDAGToDAGISel() =
delete;
366 if (
F.getFnAttribute(
"fentry-call").getValueAsString() !=
"true") {
367 if (
F.hasFnAttribute(
"mnop-mcount"))
369 if (
F.hasFnAttribute(
"mrecord-mcount"))
381 std::vector<SDValue> &OutOps)
override;
386 #include "SystemZGenDAGISel.inc"
395 ID, std::make_unique<SystemZDAGToDAGISel>(TM, OptLevel)) {}
399char SystemZDAGToDAGISelLegacy::ID = 0;
405 return new SystemZDAGToDAGISelLegacy(TM, OptLevel);
411static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
413 case SystemZAddressingMode::Disp12Only:
414 return isUInt<12>(Val);
416 case SystemZAddressingMode::Disp12Pair:
417 case SystemZAddressingMode::Disp20Only:
418 case SystemZAddressingMode::Disp20Pair:
419 return isInt<20>(Val);
421 case SystemZAddressingMode::Disp20Only128:
422 return isInt<20>(Val) && isInt<20>(Val + 8);
442 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
444 AM.IncludesDynAlloc =
true;
454 if (AM.hasIndexField() && !AM.Index.getNode()) {
464static bool expandDisp(SystemZAddressingMode &AM,
bool IsBase,
467 int64_t TestDisp = AM.Disp + Op1;
479bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
482 unsigned Opcode =
N.getOpcode();
484 if (Opcode ==
ISD::TRUNCATE &&
N.getOperand(0).getValueSizeInBits() <= 64) {
486 Opcode =
N.getOpcode();
488 if (Opcode ==
ISD::ADD || CurDAG->isBaseWithConstantOffset(
N)) {
502 cast<ConstantSDNode>(Op0)->getSExtValue());
505 cast<ConstantSDNode>(Op1)->getSExtValue());
515 cast<GlobalAddressSDNode>(Anchor)->getOffset());
523static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
526 case SystemZAddressingMode::Disp12Only:
527 case SystemZAddressingMode::Disp20Only:
528 case SystemZAddressingMode::Disp20Only128:
531 case SystemZAddressingMode::Disp12Pair:
533 return isUInt<12>(Val);
535 case SystemZAddressingMode::Disp20Pair:
537 return !isUInt<12>(Val);
561 if (isUInt<12>(Disp))
566 if (!isInt<16>(Disp))
575 if (
Index->hasOneUse())
580 unsigned IndexOpcode =
Index->getOpcode();
588 if (
Base->hasOneUse())
595bool SystemZDAGToDAGISel::selectAddress(
SDValue Addr,
596 SystemZAddressingMode &AM)
const {
604 cast<ConstantSDNode>(
Addr)->getSExtValue()))
612 while (expandAddress(AM,
true) ||
613 (AM.Index.getNode() && expandAddress(AM,
false)))
617 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
618 !
shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
626 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
639 if (
N->getNodeId() == -1 ||
652void SystemZDAGToDAGISel::getAddressOperands(
const SystemZAddressingMode &AM,
658 Base = CurDAG->getRegister(0, VT);
662 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
663 }
else if (
Base.getValueType() != VT) {
665 assert(VT == MVT::i32 &&
Base.getValueType() == MVT::i64 &&
666 "Unexpected truncation");
674 Disp = CurDAG->getTargetConstant(AM.Disp,
SDLoc(
Base), VT);
677void SystemZDAGToDAGISel::getAddressOperands(
const SystemZAddressingMode &AM,
681 getAddressOperands(AM, VT,
Base, Disp);
684 if (!
Index.getNode())
686 Index = CurDAG->getRegister(0, VT);
689bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
692 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
693 if (!selectAddress(
Addr, AM))
696 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp);
700bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
703 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
704 if (!selectAddress(
Addr, AM) || AM.Index.getNode())
707 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp);
711bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm
Form,
712 SystemZAddressingMode::DispRange DR,
715 SystemZAddressingMode AM(
Form, DR);
716 if (!selectAddress(
Addr, AM))
719 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp,
Index);
728 if (selectBDXAddr12Only(
Addr, Regs[0], Disp, Regs[1]) &&
730 for (
unsigned int I = 0;
I < 2; ++
I) {
738 Index.getOperand(1) == Elem) {
747bool SystemZDAGToDAGISel::detectOrAndInsertion(
SDValue &
Op,
755 auto *MaskNode = dyn_cast<ConstantSDNode>(
Op.getOperand(1).getNode());
760 uint64_t AndMask = MaskNode->getZExtValue();
761 if (InsertMask & AndMask)
767 if (Used != (AndMask | InsertMask)) {
768 KnownBits Known = CurDAG->computeKnownBits(
Op.getOperand(0));
773 Op =
Op.getOperand(0);
777bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
780 if (RxSBG.Rotate != 0)
781 Mask = (
Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
783 if (
TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
793 if (RxSBG.Rotate != 0)
794 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
795 return (Mask & RxSBG.Mask) != 0;
798bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG)
const {
800 unsigned Opcode =
N.getOpcode();
803 if (RxSBG.Opcode == SystemZ::RNSBG)
805 if (
N.getOperand(0).getValueSizeInBits() > 64)
807 uint64_t BitSize =
N.getValueSizeInBits();
809 if (!refineRxSBGMask(RxSBG, Mask))
811 RxSBG.Input =
N.getOperand(0);
815 if (RxSBG.Opcode == SystemZ::RNSBG)
818 auto *MaskNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
824 if (!refineRxSBGMask(RxSBG, Mask)) {
828 KnownBits Known = CurDAG->computeKnownBits(Input);
830 if (!refineRxSBGMask(RxSBG, Mask))
838 if (RxSBG.Opcode != SystemZ::RNSBG)
841 auto *MaskNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
847 if (!refineRxSBGMask(RxSBG, Mask)) {
851 KnownBits Known = CurDAG->computeKnownBits(Input);
852 Mask &= ~Known.One.getZExtValue();
853 if (!refineRxSBGMask(RxSBG, Mask))
862 if (RxSBG.BitSize != 64 ||
N.getValueType() != MVT::i64)
864 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
868 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
869 RxSBG.Input =
N.getOperand(0);
875 RxSBG.Input =
N.getOperand(0);
879 if (RxSBG.Opcode != SystemZ::RNSBG) {
881 unsigned InnerBitSize =
N.getOperand(0).getValueSizeInBits();
882 if (!refineRxSBGMask(RxSBG,
allOnes(InnerBitSize)))
885 RxSBG.Input =
N.getOperand(0);
893 unsigned BitSize =
N.getValueSizeInBits();
894 unsigned InnerBitSize =
N.getOperand(0).getValueSizeInBits();
898 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
899 RxSBG.Rotate += (BitSize - InnerBitSize);
904 RxSBG.Input =
N.getOperand(0);
909 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
913 uint64_t Count = CountNode->getZExtValue();
914 unsigned BitSize =
N.getValueSizeInBits();
915 if (Count < 1 || Count >= BitSize)
918 if (RxSBG.Opcode == SystemZ::RNSBG) {
925 if (!refineRxSBGMask(RxSBG,
allOnes(BitSize - Count) << Count))
929 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
930 RxSBG.Input =
N.getOperand(0);
936 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
940 uint64_t Count = CountNode->getZExtValue();
941 unsigned BitSize =
N.getValueSizeInBits();
942 if (Count < 1 || Count >= BitSize)
945 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode ==
ISD::SRA) {
953 if (!refineRxSBGMask(RxSBG,
allOnes(BitSize - Count)))
957 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
958 RxSBG.Input =
N.getOperand(0);
967 SDNode *
N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, VT);
973 if (
N.getValueType() == MVT::i32 && VT == MVT::i64)
974 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
975 DL, VT, getUNDEF(
DL, MVT::i64),
N);
976 if (
N.getValueType() == MVT::i64 && VT == MVT::i32)
977 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32,
DL, VT,
N);
978 assert(
N.getValueType() == VT &&
"Unexpected value types");
982bool SystemZDAGToDAGISel::tryRISBGZero(
SDNode *
N) {
984 EVT VT =
N->getValueType(0);
987 RxSBGOperands RISBG(SystemZ::RISBG,
SDValue(
N, 0));
989 while (expandRxSBG(RISBG))
996 if (Count == 0 || isa<ConstantSDNode>(RISBG.Input))
1001 if (Count == 1 &&
N->getOpcode() !=
ISD::AND)
1007 if (RISBG.Rotate == 0) {
1008 bool PreferAnd =
false;
1014 else if (RISBG.Mask == 0xff ||
1015 RISBG.Mask == 0xffff ||
1016 RISBG.Mask == 0x7fffffff ||
1022 else if (
auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
1023 if (
Load->getMemoryVT() == MVT::i32 &&
1026 RISBG.Mask == 0xffffff00 &&
1027 Subtarget->hasLoadAndZeroRightmostByte())
1037 if (
N !=
New.getNode()) {
1040 ReplaceNode(
N,
New.getNode());
1044 if (!
N->isMachineOpcode())
1050 unsigned Opcode = SystemZ::RISBG;
1052 if (Subtarget->hasMiscellaneousExtensions())
1053 Opcode = SystemZ::RISBGN;
1054 EVT OpcodeVT = MVT::i64;
1055 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1060 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1061 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1062 ((RISBG.End + RISBG.Rotate) & 63) >=
1063 ((RISBG.Start + RISBG.Rotate) & 63)) {
1064 Opcode = SystemZ::RISBMux;
1065 OpcodeVT = MVT::i32;
1070 getUNDEF(
DL, OpcodeVT),
1071 convertTo(
DL, OpcodeVT, RISBG.Input),
1072 CurDAG->getTargetConstant(RISBG.Start,
DL, MVT::i32),
1073 CurDAG->getTargetConstant(RISBG.End | 128,
DL, MVT::i32),
1074 CurDAG->getTargetConstant(RISBG.Rotate,
DL, MVT::i32)
1077 DL, VT,
SDValue(CurDAG->getMachineNode(Opcode,
DL, OpcodeVT, Ops), 0));
1078 ReplaceNode(
N,
New.getNode());
1082bool SystemZDAGToDAGISel::tryRxSBG(
SDNode *
N,
unsigned Opcode) {
1084 EVT VT =
N->getValueType(0);
1089 RxSBGOperands RxSBG[] = {
1090 RxSBGOperands(Opcode,
N->getOperand(0)),
1091 RxSBGOperands(Opcode,
N->getOperand(1))
1093 unsigned Count[] = { 0, 0 };
1094 for (
unsigned I = 0;
I < 2; ++
I)
1095 while (RxSBG[
I].Input->
hasOneUse() && expandRxSBG(RxSBG[
I]))
1107 if (Count[0] == 0 && Count[1] == 0)
1111 unsigned I = Count[0] > Count[1] ? 0 : 1;
1115 if (Opcode == SystemZ::ROSBG && (RxSBG[
I].Mask & 0xff) == 0)
1116 if (
auto *Load = dyn_cast<LoadSDNode>(Op0.
getNode()))
1117 if (
Load->getMemoryVT() == MVT::i8)
1122 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[
I].Mask)) {
1123 Opcode = SystemZ::RISBG;
1125 if (Subtarget->hasMiscellaneousExtensions())
1126 Opcode = SystemZ::RISBGN;
1130 convertTo(
DL, MVT::i64, Op0),
1131 convertTo(
DL, MVT::i64, RxSBG[
I].Input),
1132 CurDAG->getTargetConstant(RxSBG[
I].Start,
DL, MVT::i32),
1133 CurDAG->getTargetConstant(RxSBG[
I].
End,
DL, MVT::i32),
1134 CurDAG->getTargetConstant(RxSBG[
I].Rotate,
DL, MVT::i32)
1137 DL, VT,
SDValue(CurDAG->getMachineNode(Opcode,
DL, MVT::i64, Ops), 0));
1138 ReplaceNode(
N,
New.getNode());
1142void SystemZDAGToDAGISel::splitLargeImmediate(
unsigned Opcode,
SDNode *
Node,
1145 EVT VT =
Node->getValueType(0);
1165 SelectCode(
Upper.getNode());
1166 Upper = Handle.getValue();
1172 ReplaceNode(
Node,
Or.getNode());
1174 SelectCode(
Or.getNode());
1177void SystemZDAGToDAGISel::loadVectorConstant(
1184 EVT VT =
Node->getValueType(0);
1187 for (
unsigned OpVal : VCI.
OpVals)
1188 Ops.
push_back(CurDAG->getTargetConstant(OpVal,
DL, MVT::i32));
1192 ReplaceNode(
Node,
Op.getNode());
1196 SelectCode(BitCast.
getNode());
1198 unsigned SubRegIdx =
1199 (VT.
getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1201 Node, CurDAG->getTargetExtractSubreg(SubRegIdx,
DL, VT,
Op).getNode());
1203 SelectCode(
Op.getNode());
1210 SDValue CP = CurDAG->getTargetConstantPool(
1212 TLI->getPointerTy(CurDAG->getDataLayout()));
1214 EVT PtrVT =
CP.getValueType();
1216 SDValue(CurDAG->getMachineNode(SystemZ::LARL,
DL, PtrVT, CP), 0),
1217 CurDAG->getTargetConstant(0,
DL, PtrVT),
1218 CurDAG->getRegister(0, PtrVT),
1219 CurDAG->getEntryNode()
1221 ResNode = CurDAG->getMachineNode(SystemZ::VL,
DL, VT, MVT::Other, Ops);
1231 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
1235bool SystemZDAGToDAGISel::tryGather(
SDNode *
N,
unsigned Opcode) {
1237 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1241 unsigned Elem = ElemN->getZExtValue();
1242 EVT VT =
N->getValueType(0);
1246 auto *
Load = dyn_cast<LoadSDNode>(
N->getOperand(1));
1247 if (!Load || !
Load->hasNUsesOfValue(1, 0))
1249 if (
Load->getMemoryVT().getSizeInBits() !=
1250 Load->getValueType(0).getSizeInBits())
1254 if (!selectBDVAddr12Only(
Load->getBasePtr(), ElemV,
Base, Disp,
Index) ||
1261 CurDAG->getTargetConstant(Elem,
DL, MVT::i32),
Load->getChain()
1263 SDNode *Res = CurDAG->getMachineNode(Opcode,
DL, VT, MVT::Other, Ops);
1265 ReplaceNode(
N, Res);
1269bool SystemZDAGToDAGISel::tryScatter(
StoreSDNode *Store,
unsigned Opcode) {
1273 if (
Store->getMemoryVT().getSizeInBits() !=
Value.getValueSizeInBits())
1277 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1283 unsigned Elem = ElemN->getZExtValue();
1288 if (!selectBDVAddr12Only(
Store->getBasePtr(), ElemV,
Base, Disp,
Index) ||
1294 Vec,
Base, Disp,
Index, CurDAG->getTargetConstant(Elem,
DL, MVT::i32),
1297 ReplaceNode(Store, CurDAG->getMachineNode(Opcode,
DL, MVT::Other, Ops));
1325 LoadNode = cast<LoadSDNode>(Load);
1328 if (!Load.hasOneUse())
1340 bool ChainCheck =
false;
1341 if (Chain == Load.getValue(1)) {
1348 const unsigned int Max = 1024;
1351 if (
Op == Load.getValue(1)) {
1364 if (
Op.getNode() != LoadNode)
1375 MVT::Other, ChainOps);
1392bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(
SDNode *
Node) {
1402 unsigned NewOpc = 0;
1403 bool NegateOperand =
false;
1408 NegateOperand =
true;
1411 if (MemVT == MVT::i32)
1412 NewOpc = SystemZ::ASI;
1413 else if (MemVT == MVT::i64)
1414 NewOpc = SystemZ::AGSI;
1419 NegateOperand =
true;
1422 if (MemVT == MVT::i32)
1423 NewOpc = SystemZ::ALSI;
1424 else if (MemVT == MVT::i64)
1425 NewOpc = SystemZ::ALGSI;
1438 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1441 auto OperandV = OperandC->getAPIntValue();
1443 OperandV = -OperandV;
1444 if (OperandV.getSignificantBits() > 8)
1446 Operand = CurDAG->getTargetConstant(OperandV,
DL, MemVT);
1452 SDValue Ops[] = {
Base, Disp, Operand, InputChain };
1454 CurDAG->getMachineNode(NewOpc,
DL, MVT::i32, MVT::Other, Ops);
1455 CurDAG->setNodeMemRefs(
1460 CurDAG->RemoveDeadNode(
Node);
1464bool SystemZDAGToDAGISel::canUseBlockOperation(
StoreSDNode *Store,
1467 if (
Load->getMemoryVT() !=
Store->getMemoryVT())
1471 if (
Load->isVolatile() ||
Store->isVolatile())
1475 if (
Load->isInvariant() &&
Load->isDereferenceable())
1479 const Value *V1 =
Load->getMemOperand()->getValue();
1486 int64_t End1 =
Load->getSrcValueOffset() +
Size;
1487 int64_t End2 =
Store->getSrcValueOffset() +
Size;
1488 if (V1 == V2 && End1 == End2)
1495bool SystemZDAGToDAGISel::storeLoadCanUseMVC(
SDNode *
N)
const {
1496 auto *
Store = cast<StoreSDNode>(
N);
1497 auto *
Load = cast<LoadSDNode>(
Store->getValue());
1511 return canUseBlockOperation(Store, Load);
1514bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(
SDNode *
N,
1516 auto *StoreA = cast<StoreSDNode>(
N);
1517 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 -
I));
1518 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(
I));
1519 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1520 canUseBlockOperation(StoreA, LoadB);
1523bool SystemZDAGToDAGISel::storeLoadIsAligned(
SDNode *
N)
const {
1525 auto *MemAccess = cast<MemSDNode>(
N);
1526 auto *LdSt = dyn_cast<LSBaseSDNode>(MemAccess);
1527 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1530 assert(MMO &&
"Expected a memory operand.");
1534 if (MemAccess->getAlign().value() < StoreSize ||
1535 (LdSt && !LdSt->getOffset().isUndef()))
1544 if ((PSV->isGOT() || PSV->isConstantPool()))
1550 dyn_cast<GlobalAddressSDNode>(
BasePtr.getOperand(0))) {
1552 if (GA->getOffset() % StoreSize != 0)
1567 if (
auto *L = dyn_cast<LoadSDNode>(
N))
1568 ETy =
L->getExtensionType();
1569 else if (
auto *AL = dyn_cast<AtomicSDNode>(
N))
1570 ETy =
AL->getExtensionType();
1576void SystemZDAGToDAGISel::Select(
SDNode *
Node) {
1578 if (
Node->isMachineOpcode()) {
1580 Node->setNodeId(-1);
1584 unsigned Opcode =
Node->getOpcode();
1588 if (tryRxSBG(
Node, SystemZ::ROSBG))
1594 if (tryRxSBG(
Node, SystemZ::RXSBG))
1601 if (
Node->getValueType(0) == MVT::i64 &&
1603 if (
auto *Op1 = dyn_cast<ConstantSDNode>(
Node->getOperand(1))) {
1604 uint64_t Val = Op1->getZExtValue();
1607 if (Subtarget->hasMiscellaneousExtensions3()) {
1608 unsigned ChildOpcode =
Node->getOperand(0).getOpcode();
1617 auto Op0 =
Node->getOperand(0);
1618 if (
auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->
getOperand(1)))
1619 if (Op0Op1->getZExtValue() == (
uint64_t)-1)
1624 if (Opcode ==
ISD::XOR && Op1->isAllOnes())
1627 splitLargeImmediate(Opcode,
Node,
Node->getOperand(0),
1636 if (tryRxSBG(
Node, SystemZ::RNSBG))
1643 if (tryRISBGZero(
Node))
1648 if (
Node->getValueType(0) == MVT::i128) {
1653 uint64_t Bytes[2] = { 0x0706050403020100ULL, 0x0f0e0d0c0b0a0908ULL };
1657 MVT::v16i8, Ops), 0);
1661 ReplaceNode(
Node, ResNode);
1662 SelectCode(Src.getNode());
1663 SelectCode(ResNode);
1671 if (
Node->getValueType(0) == MVT::i64) {
1679 if (
Node->getValueType(0) == MVT::i128) {
1680 const APInt &Val =
Node->getAsAPIntVal();
1683 loadVectorConstant(VCI,
Node);
1687 SDNode *ResNode = loadPoolVectorConstant(Val, MVT::i128,
SDLoc(
Node));
1688 ReplaceNode(
Node, ResNode);
1699 (Subtarget->hasLoadStoreOnCond2() &&
1700 Node->getValueType(0).isInteger() &&
1701 Node->getValueType(0).getSizeInBits() <= 64 &&
1703 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1705 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1711 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1715 CurDAG->UpdateNodeOperands(
Node, Op1, Op0, CCValid, CCMask, Op4);
1716 if (UpdatedNode !=
Node) {
1718 ReplaceNode(
Node, UpdatedNode);
1726 EVT VT =
Node->getValueType(0);
1728 if (ElemBitSize == 32) {
1729 if (tryGather(
Node, SystemZ::VGEF))
1731 }
else if (ElemBitSize == 64) {
1732 if (tryGather(
Node, SystemZ::VGEG))
1739 auto *BVN = cast<BuildVectorSDNode>(
Node);
1742 loadVectorConstant(VCI,
Node);
1750 if (
Imm.isZero() ||
Imm.isNegZero())
1755 loadVectorConstant(VCI,
Node);
1760 if (tryFoldLoadStoreIntoMemOperand(
Node))
1763 unsigned ElemBitSize =
Store->getValue().getValueSizeInBits();
1764 if (ElemBitSize == 32) {
1765 if (tryScatter(Store, SystemZ::VSCEF))
1767 }
else if (ElemBitSize == 64) {
1768 if (tryScatter(Store, SystemZ::VSCEG))
1775 auto *AtomOp = cast<AtomicSDNode>(
Node);
1779 StoreSDNode *St = cast<StoreSDNode>(CurDAG->getTruncStore(
1780 AtomOp->getChain(),
SDLoc(AtomOp), AtomOp->getVal(),
1781 AtomOp->getBasePtr(), AtomOp->getMemoryVT(), AtomOp->getMemOperand()));
1786 if (AtomOp->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent)
1787 Chain = CurDAG->getMachineNode(SystemZ::Serialize,
SDLoc(AtomOp),
1788 MVT::Other,
SDValue(Chain, 0));
1789 ReplaceNode(
Node, Chain);
1798bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand(
1800 std::vector<SDValue> &OutOps) {
1801 SystemZAddressingMode::AddrForm
Form;
1802 SystemZAddressingMode::DispRange DispRange;
1805 switch(ConstraintID) {
1808 case InlineAsm::ConstraintCode::i:
1809 case InlineAsm::ConstraintCode::Q:
1810 case InlineAsm::ConstraintCode::ZQ:
1812 Form = SystemZAddressingMode::FormBD;
1813 DispRange = SystemZAddressingMode::Disp12Only;
1815 case InlineAsm::ConstraintCode::R:
1816 case InlineAsm::ConstraintCode::ZR:
1818 Form = SystemZAddressingMode::FormBDXNormal;
1819 DispRange = SystemZAddressingMode::Disp12Only;
1821 case InlineAsm::ConstraintCode::S:
1822 case InlineAsm::ConstraintCode::ZS:
1824 Form = SystemZAddressingMode::FormBD;
1825 DispRange = SystemZAddressingMode::Disp20Only;
1827 case InlineAsm::ConstraintCode::T:
1828 case InlineAsm::ConstraintCode::m:
1829 case InlineAsm::ConstraintCode::o:
1830 case InlineAsm::ConstraintCode::p:
1831 case InlineAsm::ConstraintCode::ZT:
1836 Form = SystemZAddressingMode::FormBDXNormal;
1837 DispRange = SystemZAddressingMode::Disp20Only;
1843 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1845 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(),
DL, MVT::i32);
1852 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1860 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1865 OutOps.push_back(
Base);
1866 OutOps.push_back(Disp);
1867 OutOps.push_back(
Index);
1882 if (!
N.hasOneUse() || !
U->hasOneUse())
1889 SDNode *CCUser = *
U->use_begin();
1890 SDNode *CCRegUser =
nullptr;
1892 cast<RegisterSDNode>(CCUser->
getOperand(1))->getReg() == SystemZ::CC) {
1893 for (
auto *U : CCUser->
uses()) {
1894 if (CCRegUser ==
nullptr)
1896 else if (CCRegUser != U)
1900 if (CCRegUser ==
nullptr)
1913 if (!IsLegalToFold(
N, U, CCRegUser, OptLevel,
false))
1923struct IPMConversion {
1924 IPMConversion(
unsigned xorValue, int64_t addValue,
unsigned bit)
1925 : XORValue(xorValue), AddValue(addValue),
Bit(bit) {}
2001 auto *TrueOp = dyn_cast<ConstantSDNode>(
Node->getOperand(0));
2002 auto *FalseOp = dyn_cast<ConstantSDNode>(
Node->getOperand(1));
2003 if (!TrueOp || !FalseOp)
2005 if (FalseOp->getZExtValue() != 0)
2007 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
2010 auto *CCValidOp = dyn_cast<ConstantSDNode>(
Node->getOperand(2));
2011 auto *CCMaskOp = dyn_cast<ConstantSDNode>(
Node->getOperand(3));
2012 if (!CCValidOp || !CCMaskOp)
2014 int CCValid = CCValidOp->getZExtValue();
2015 int CCMask = CCMaskOp->getZExtValue();
2024 CurDAG->getConstant(
IPM.XORValue,
DL, MVT::i32));
2028 CurDAG->getConstant(
IPM.AddValue,
DL, MVT::i32));
2030 EVT VT =
Node->getValueType(0);
2031 if (VT == MVT::i32 &&
IPM.Bit == 31) {
2033 Result = CurDAG->getNode(ShiftOp,
DL, MVT::i32, Result,
2034 CurDAG->getConstant(
IPM.Bit,
DL, MVT::i32));
2039 if (TrueOp->getSExtValue() == 1) {
2042 CurDAG->getConstant(
IPM.Bit,
DL, MVT::i32));
2044 CurDAG->getConstant(1,
DL, VT));
2050 CurDAG->getConstant(ShlAmt,
DL, MVT::i32));
2052 CurDAG->getConstant(SraAmt,
DL, MVT::i32));
2059bool SystemZDAGToDAGISel::shouldSelectForReassoc(
SDNode *
N)
const {
2060 EVT VT =
N->getValueType(0);
2062 return N->getFlags().hasAllowReassociation() &&
2063 N->getFlags().hasNoSignedZeros() && Subtarget->hasVector() &&
2064 (VT != MVT::f32 || Subtarget->hasVectorEnhancements1()) &&
2065 !
N->isStrictFPOpcode();
2068void SystemZDAGToDAGISel::PreprocessISelDAG() {
2071 if (Subtarget->hasLoadStoreOnCond2())
2074 bool MadeChange =
false;
2077 E = CurDAG->allnodes_end();
2084 switch (
N->getOpcode()) {
2087 Res = expandSelectBoolean(
N);
2092 LLVM_DEBUG(
dbgs() <<
"SystemZ DAG preprocessing replacing:\nOld: ");
2098 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
2104 CurDAG->RemoveDeadNodes();
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
amdgpu AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
const HexagonInstrInfo * TII
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask)
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
static uint64_t allOnes(unsigned int Count)
DEMANGLE_DUMP_METHOD void dump() const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
const DataLayout & getDataLayout() const
Get the data layout of the module this global belongs to.
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
bool isAtomic() const
Returns true if this operation has an atomic ordering requirement of unordered or higher,...
@ MOLoad
The memory access reads data.
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
An SDNode that represents everything that will be needed to construct a MachineInstr.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
bool isNonTemporal() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
Special value supplied for machine level alias analysis.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
static int getUninvalidatedNodeId(SDNode *N)
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
virtual bool runOnMachineFunction(MachineFunction &mf)
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SystemZInstrInfo * getInstrInfo() const override
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
static IntegerType * getInt128Ty(LLVMContext &C)
LLVM Value Representation.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ BSWAP
Byte Swap and Counting operators.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SHL
Shift and rotation operations.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isPCREL(unsigned Opcode)
static bool isImmHF(uint64_t Val)
static bool isImmLF(uint64_t Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CodeGenOptLevel
Code generation optimization level.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Or
Bitwise or logical OR of integers.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)