23#define DEBUG_TYPE "systemz-isel"
24#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
28struct SystemZAddressingMode {
63 bool IncludesDynAlloc;
65 SystemZAddressingMode(AddrForm form, DispRange dr)
66 :
Form(form), DR(dr), Disp(0), IncludesDynAlloc(
false) {}
69 bool hasIndexField() {
return Form != FormBD; }
72 bool isDynAlloc() {
return Form == FormBDXDynAlloc; }
75 errs() <<
"SystemZAddressingMode " <<
this <<
'\n';
79 Base.getNode()->dump(DAG);
83 if (hasIndexField()) {
86 Index.getNode()->dump(DAG);
91 errs() <<
" Disp " << Disp;
93 errs() <<
" + ADJDYNALLOC";
119struct RxSBGOperands {
120 RxSBGOperands(
unsigned Op,
SDValue N)
121 : Opcode(Op), BitSize(
N.getValueSizeInBits()),
122 Mask(
allOnes(BitSize)), Input(
N), Start(64 - BitSize),
End(63),
139 return CurDAG->getTargetConstant(Imm,
SDLoc(
Node),
Node->getValueType(0));
152 bool expandAddress(SystemZAddressingMode &AM,
bool IsBase)
const;
155 bool selectAddress(
SDValue N, SystemZAddressingMode &AM)
const;
158 void getAddressOperands(
const SystemZAddressingMode &AM,
EVT VT,
160 void getAddressOperands(
const SystemZAddressingMode &AM,
EVT VT,
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR,
SDValue Addr,
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR,
SDValue Addr,
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm
Form,
193 return selectBDAddr(SystemZAddressingMode::Disp12Only,
Addr,
Base, Disp);
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair,
Addr,
Base, Disp);
199 return selectBDAddr(SystemZAddressingMode::Disp20Only,
Addr,
Base, Disp);
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair,
Addr,
Base, Disp);
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair,
Addr,
Base, Disp);
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair,
Addr,
Base, Disp);
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
277 bool refineRxSBGMask(RxSBGOperands &RxSBG,
uint64_t Mask)
const;
281 bool expandRxSBG(RxSBGOperands &RxSBG)
const;
295 bool tryRxSBG(
SDNode *
N,
unsigned Opcode);
311 bool tryGather(
SDNode *
N,
unsigned Opcode);
314 bool tryScatter(
StoreSDNode *Store,
unsigned Opcode);
319 bool tryFoldLoadStoreIntoMemOperand(
SDNode *
Node);
334 bool storeLoadCanUseMVC(
SDNode *
N)
const;
339 bool storeLoadCanUseBlockBinary(
SDNode *
N,
unsigned I)
const;
343 bool storeLoadIsAligned(
SDNode *
N)
const;
351 SystemZDAGToDAGISel() =
delete;
358 if (
F.getFnAttribute(
"fentry-call").getValueAsString() !=
"true") {
359 if (
F.hasFnAttribute(
"mnop-mcount"))
361 if (
F.hasFnAttribute(
"mrecord-mcount"))
372 std::vector<SDValue> &OutOps)
override;
377 #include "SystemZGenDAGISel.inc"
381char SystemZDAGToDAGISel::ID = 0;
386 CodeGenOpt::Level OptLevel) {
387 return new SystemZDAGToDAGISel(
TM, OptLevel);
393static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
395 case SystemZAddressingMode::Disp12Only:
396 return isUInt<12>(Val);
398 case SystemZAddressingMode::Disp12Pair:
399 case SystemZAddressingMode::Disp20Only:
400 case SystemZAddressingMode::Disp20Pair:
401 return isInt<20>(Val);
403 case SystemZAddressingMode::Disp20Only128:
404 return isInt<20>(Val) && isInt<20>(Val + 8);
424 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
426 AM.IncludesDynAlloc =
true;
436 if (AM.hasIndexField() && !AM.Index.getNode()) {
446static bool expandDisp(SystemZAddressingMode &AM,
bool IsBase,
449 int64_t TestDisp = AM.Disp + Op1;
461bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
464 unsigned Opcode =
N.getOpcode();
467 Opcode =
N.getOpcode();
469 if (Opcode ==
ISD::ADD || CurDAG->isBaseWithConstantOffset(
N)) {
483 cast<ConstantSDNode>(Op0)->getSExtValue());
486 cast<ConstantSDNode>(Op1)->getSExtValue());
496 cast<GlobalAddressSDNode>(Anchor)->getOffset());
504static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
507 case SystemZAddressingMode::Disp12Only:
508 case SystemZAddressingMode::Disp20Only:
509 case SystemZAddressingMode::Disp20Only128:
512 case SystemZAddressingMode::Disp12Pair:
514 return isUInt<12>(Val);
516 case SystemZAddressingMode::Disp20Pair:
518 return !isUInt<12>(Val);
542 if (isUInt<12>(Disp))
547 if (!isInt<16>(Disp))
556 if (
Index->hasOneUse())
561 unsigned IndexOpcode =
Index->getOpcode();
569 if (
Base->hasOneUse())
576bool SystemZDAGToDAGISel::selectAddress(
SDValue Addr,
577 SystemZAddressingMode &AM)
const {
585 cast<ConstantSDNode>(
Addr)->getSExtValue()))
593 while (expandAddress(AM,
true) ||
594 (AM.Index.getNode() && expandAddress(AM,
false)))
598 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
599 !
shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
607 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
620 if (
N->getNodeId() == -1 ||
633void SystemZDAGToDAGISel::getAddressOperands(
const SystemZAddressingMode &AM,
639 Base = CurDAG->getRegister(0, VT);
643 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
644 }
else if (
Base.getValueType() != VT) {
646 assert(VT == MVT::i32 &&
Base.getValueType() == MVT::i64 &&
647 "Unexpected truncation");
655 Disp = CurDAG->getTargetConstant(AM.Disp,
SDLoc(
Base), VT);
658void SystemZDAGToDAGISel::getAddressOperands(
const SystemZAddressingMode &AM,
662 getAddressOperands(AM, VT,
Base, Disp);
665 if (!
Index.getNode())
667 Index = CurDAG->getRegister(0, VT);
670bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
673 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
674 if (!selectAddress(
Addr, AM))
677 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp);
681bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
684 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
685 if (!selectAddress(
Addr, AM) || AM.Index.getNode())
688 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp);
692bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm
Form,
693 SystemZAddressingMode::DispRange DR,
696 SystemZAddressingMode AM(
Form, DR);
697 if (!selectAddress(
Addr, AM))
700 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp,
Index);
709 if (selectBDXAddr12Only(
Addr, Regs[0], Disp, Regs[1]) &&
710 Regs[0].getNode() && Regs[1].getNode()) {
711 for (
unsigned int I = 0;
I < 2; ++
I) {
719 Index.getOperand(1) == Elem) {
728bool SystemZDAGToDAGISel::detectOrAndInsertion(
SDValue &Op,
736 auto *MaskNode = dyn_cast<ConstantSDNode>(
Op.getOperand(1).getNode());
741 uint64_t AndMask = MaskNode->getZExtValue();
742 if (InsertMask & AndMask)
748 if (Used != (AndMask | InsertMask)) {
749 KnownBits Known = CurDAG->computeKnownBits(
Op.getOperand(0));
754 Op =
Op.getOperand(0);
758bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
761 if (RxSBG.Rotate != 0)
762 Mask = (
Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
764 if (
TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
774 if (RxSBG.Rotate != 0)
775 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
776 return (Mask & RxSBG.Mask) != 0;
779bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG)
const {
781 unsigned Opcode =
N.getOpcode();
784 if (RxSBG.Opcode == SystemZ::RNSBG)
786 uint64_t BitSize =
N.getValueSizeInBits();
788 if (!refineRxSBGMask(RxSBG, Mask))
790 RxSBG.Input =
N.getOperand(0);
794 if (RxSBG.Opcode == SystemZ::RNSBG)
797 auto *MaskNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
803 if (!refineRxSBGMask(RxSBG, Mask)) {
807 KnownBits Known = CurDAG->computeKnownBits(Input);
809 if (!refineRxSBGMask(RxSBG, Mask))
817 if (RxSBG.Opcode != SystemZ::RNSBG)
820 auto *MaskNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
826 if (!refineRxSBGMask(RxSBG, Mask)) {
830 KnownBits Known = CurDAG->computeKnownBits(Input);
831 Mask &= ~Known.One.getZExtValue();
832 if (!refineRxSBGMask(RxSBG, Mask))
841 if (RxSBG.BitSize != 64 ||
N.getValueType() != MVT::i64)
843 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
847 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
848 RxSBG.Input =
N.getOperand(0);
854 RxSBG.Input =
N.getOperand(0);
858 if (RxSBG.Opcode != SystemZ::RNSBG) {
860 unsigned InnerBitSize =
N.getOperand(0).getValueSizeInBits();
861 if (!refineRxSBGMask(RxSBG,
allOnes(InnerBitSize)))
864 RxSBG.Input =
N.getOperand(0);
872 unsigned BitSize =
N.getValueSizeInBits();
873 unsigned InnerBitSize =
N.getOperand(0).getValueSizeInBits();
877 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
878 RxSBG.Rotate += (BitSize - InnerBitSize);
883 RxSBG.Input =
N.getOperand(0);
888 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
892 uint64_t Count = CountNode->getZExtValue();
893 unsigned BitSize =
N.getValueSizeInBits();
894 if (Count < 1 || Count >= BitSize)
897 if (RxSBG.Opcode == SystemZ::RNSBG) {
904 if (!refineRxSBGMask(RxSBG,
allOnes(BitSize - Count) << Count))
908 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
909 RxSBG.Input =
N.getOperand(0);
915 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
919 uint64_t Count = CountNode->getZExtValue();
920 unsigned BitSize =
N.getValueSizeInBits();
921 if (Count < 1 || Count >= BitSize)
924 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode ==
ISD::SRA) {
932 if (!refineRxSBGMask(RxSBG,
allOnes(BitSize - Count)))
936 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
937 RxSBG.Input =
N.getOperand(0);
946 SDNode *
N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, VT);
952 if (
N.getValueType() == MVT::i32 && VT == MVT::i64)
953 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
954 DL, VT, getUNDEF(
DL, MVT::i64),
N);
955 if (
N.getValueType() == MVT::i64 && VT == MVT::i32)
956 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32,
DL, VT,
N);
957 assert(
N.getValueType() == VT &&
"Unexpected value types");
961bool SystemZDAGToDAGISel::tryRISBGZero(
SDNode *
N) {
963 EVT VT =
N->getValueType(0);
966 RxSBGOperands RISBG(SystemZ::RISBG,
SDValue(
N, 0));
968 while (expandRxSBG(RISBG))
975 if (Count == 0 || isa<ConstantSDNode>(RISBG.Input))
980 if (Count == 1 &&
N->getOpcode() !=
ISD::AND)
986 if (RISBG.Rotate == 0) {
987 bool PreferAnd =
false;
993 else if (RISBG.Mask == 0xff ||
994 RISBG.Mask == 0xffff ||
995 RISBG.Mask == 0x7fffffff ||
1001 else if (
auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
1002 if (
Load->getMemoryVT() == MVT::i32 &&
1005 RISBG.Mask == 0xffffff00 &&
1006 Subtarget->hasLoadAndZeroRightmostByte())
1016 if (
N !=
New.getNode()) {
1019 ReplaceNode(
N,
New.getNode());
1023 if (!
N->isMachineOpcode())
1029 unsigned Opcode = SystemZ::RISBG;
1031 if (Subtarget->hasMiscellaneousExtensions())
1032 Opcode = SystemZ::RISBGN;
1033 EVT OpcodeVT = MVT::i64;
1034 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1039 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1040 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1041 ((RISBG.End + RISBG.Rotate) & 63) >=
1042 ((RISBG.Start + RISBG.Rotate) & 63)) {
1043 Opcode = SystemZ::RISBMux;
1044 OpcodeVT = MVT::i32;
1049 getUNDEF(
DL, OpcodeVT),
1050 convertTo(
DL, OpcodeVT, RISBG.Input),
1051 CurDAG->getTargetConstant(RISBG.Start,
DL, MVT::i32),
1052 CurDAG->getTargetConstant(RISBG.End | 128,
DL, MVT::i32),
1053 CurDAG->getTargetConstant(RISBG.Rotate,
DL, MVT::i32)
1056 DL, VT,
SDValue(CurDAG->getMachineNode(Opcode,
DL, OpcodeVT, Ops), 0));
1057 ReplaceNode(
N,
New.getNode());
1061bool SystemZDAGToDAGISel::tryRxSBG(
SDNode *
N,
unsigned Opcode) {
1063 EVT VT =
N->getValueType(0);
1068 RxSBGOperands RxSBG[] = {
1069 RxSBGOperands(Opcode,
N->getOperand(0)),
1070 RxSBGOperands(Opcode,
N->getOperand(1))
1072 unsigned Count[] = { 0, 0 };
1073 for (
unsigned I = 0;
I < 2; ++
I)
1074 while (RxSBG[
I].Input->
hasOneUse() && expandRxSBG(RxSBG[
I]))
1086 if (Count[0] == 0 && Count[1] == 0)
1090 unsigned I = Count[0] > Count[1] ? 0 : 1;
1094 if (Opcode == SystemZ::ROSBG && (RxSBG[
I].Mask & 0xff) == 0)
1095 if (
auto *Load = dyn_cast<LoadSDNode>(Op0.
getNode()))
1096 if (
Load->getMemoryVT() == MVT::i8)
1101 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[
I].Mask)) {
1102 Opcode = SystemZ::RISBG;
1104 if (Subtarget->hasMiscellaneousExtensions())
1105 Opcode = SystemZ::RISBGN;
1109 convertTo(
DL, MVT::i64, Op0),
1110 convertTo(
DL, MVT::i64, RxSBG[
I].Input),
1111 CurDAG->getTargetConstant(RxSBG[
I].Start,
DL, MVT::i32),
1112 CurDAG->getTargetConstant(RxSBG[
I].
End,
DL, MVT::i32),
1113 CurDAG->getTargetConstant(RxSBG[
I].Rotate,
DL, MVT::i32)
1116 DL, VT,
SDValue(CurDAG->getMachineNode(Opcode,
DL, MVT::i64, Ops), 0));
1117 ReplaceNode(
N,
New.getNode());
1121void SystemZDAGToDAGISel::splitLargeImmediate(
unsigned Opcode,
SDNode *
Node,
1124 EVT VT =
Node->getValueType(0);
1144 SelectCode(
Upper.getNode());
1145 Upper = Handle.getValue();
1151 ReplaceNode(
Node,
Or.getNode());
1153 SelectCode(
Or.getNode());
1156void SystemZDAGToDAGISel::loadVectorConstant(
1163 EVT VT =
Node->getValueType(0);
1166 for (
unsigned OpVal : VCI.
OpVals)
1167 Ops.
push_back(CurDAG->getTargetConstant(OpVal,
DL, MVT::i32));
1171 ReplaceNode(
Node,
Op.getNode());
1175 SelectCode(BitCast.
getNode());
1177 unsigned SubRegIdx =
1178 (VT.
getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1180 Node, CurDAG->getTargetExtractSubreg(SubRegIdx,
DL, VT, Op).getNode());
1182 SelectCode(
Op.getNode());
1185bool SystemZDAGToDAGISel::tryGather(
SDNode *
N,
unsigned Opcode) {
1187 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1191 unsigned Elem = ElemN->getZExtValue();
1192 EVT VT =
N->getValueType(0);
1196 auto *
Load = dyn_cast<LoadSDNode>(
N->getOperand(1));
1197 if (!Load || !
Load->hasNUsesOfValue(1, 0))
1199 if (
Load->getMemoryVT().getSizeInBits() !=
1200 Load->getValueType(0).getSizeInBits())
1204 if (!selectBDVAddr12Only(
Load->getBasePtr(), ElemV,
Base, Disp,
Index) ||
1211 CurDAG->getTargetConstant(Elem,
DL, MVT::i32),
Load->getChain()
1213 SDNode *Res = CurDAG->getMachineNode(Opcode,
DL, VT, MVT::Other, Ops);
1215 ReplaceNode(
N, Res);
1219bool SystemZDAGToDAGISel::tryScatter(
StoreSDNode *Store,
unsigned Opcode) {
1223 if (
Store->getMemoryVT().getSizeInBits() !=
Value.getValueSizeInBits())
1227 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1233 unsigned Elem = ElemN->getZExtValue();
1238 if (!selectBDVAddr12Only(
Store->getBasePtr(), ElemV,
Base, Disp,
Index) ||
1244 Vec,
Base, Disp,
Index, CurDAG->getTargetConstant(Elem,
DL, MVT::i32),
1247 ReplaceNode(Store, CurDAG->getMachineNode(Opcode,
DL, MVT::Other, Ops));
1275 LoadNode = cast<LoadSDNode>(Load);
1278 if (!Load.hasOneUse())
1290 bool ChainCheck =
false;
1291 if (Chain == Load.getValue(1)) {
1298 const unsigned int Max = 1024;
1301 if (Op == Load.getValue(1)) {
1314 if (Op.getNode() != LoadNode)
1325 MVT::Other, ChainOps);
1342bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(
SDNode *
Node) {
1352 unsigned NewOpc = 0;
1353 bool NegateOperand =
false;
1358 NegateOperand =
true;
1361 if (MemVT == MVT::i32)
1362 NewOpc = SystemZ::ASI;
1363 else if (MemVT == MVT::i64)
1364 NewOpc = SystemZ::AGSI;
1369 NegateOperand =
true;
1372 if (MemVT == MVT::i32)
1373 NewOpc = SystemZ::ALSI;
1374 else if (MemVT == MVT::i64)
1375 NewOpc = SystemZ::ALGSI;
1388 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1391 auto OperandV = OperandC->getAPIntValue();
1393 OperandV = -OperandV;
1394 if (OperandV.getSignificantBits() > 8)
1396 Operand = CurDAG->getTargetConstant(OperandV,
DL, MemVT);
1402 SDValue Ops[] = {
Base, Disp, Operand, InputChain };
1404 CurDAG->getMachineNode(NewOpc,
DL, MVT::i32, MVT::Other, Ops);
1405 CurDAG->setNodeMemRefs(
1410 CurDAG->RemoveDeadNode(
Node);
1414bool SystemZDAGToDAGISel::canUseBlockOperation(
StoreSDNode *Store,
1417 if (
Load->getMemoryVT() !=
Store->getMemoryVT())
1421 if (
Load->isVolatile() ||
Store->isVolatile())
1425 if (
Load->isInvariant() &&
Load->isDereferenceable())
1429 const Value *V1 =
Load->getMemOperand()->getValue();
1436 int64_t End1 =
Load->getSrcValueOffset() +
Size;
1437 int64_t End2 =
Store->getSrcValueOffset() +
Size;
1438 if (V1 == V2 && End1 == End2)
1445bool SystemZDAGToDAGISel::storeLoadCanUseMVC(
SDNode *
N)
const {
1446 auto *
Store = cast<StoreSDNode>(
N);
1447 auto *
Load = cast<LoadSDNode>(
Store->getValue());
1461 return canUseBlockOperation(Store, Load);
1464bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(
SDNode *
N,
1466 auto *StoreA = cast<StoreSDNode>(
N);
1467 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 -
I));
1468 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(
I));
1469 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1470 canUseBlockOperation(StoreA, LoadB);
1473bool SystemZDAGToDAGISel::storeLoadIsAligned(
SDNode *
N)
const {
1475 auto *MemAccess = cast<LSBaseSDNode>(
N);
1476 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1479 assert(MMO &&
"Expected a memory operand.");
1482 if (MemAccess->getAlign().value() < StoreSize ||
1483 !MemAccess->getOffset().isUndef())
1492 if ((PSV->isGOT() || PSV->isConstantPool()))
1498 dyn_cast<GlobalAddressSDNode>(
BasePtr.getOperand(0))) {
1500 if (GA->getOffset() % StoreSize != 0)
1513void SystemZDAGToDAGISel::Select(
SDNode *
Node) {
1515 if (
Node->isMachineOpcode()) {
1517 Node->setNodeId(-1);
1521 unsigned Opcode =
Node->getOpcode();
1525 if (tryRxSBG(
Node, SystemZ::ROSBG))
1531 if (tryRxSBG(
Node, SystemZ::RXSBG))
1538 if (
Node->getValueType(0) == MVT::i64 &&
1540 if (
auto *Op1 = dyn_cast<ConstantSDNode>(
Node->getOperand(1))) {
1541 uint64_t Val = Op1->getZExtValue();
1544 if (Subtarget->hasMiscellaneousExtensions3()) {
1545 unsigned ChildOpcode =
Node->getOperand(0).getOpcode();
1554 auto Op0 =
Node->getOperand(0);
1555 if (
auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->
getOperand(1)))
1556 if (Op0Op1->getZExtValue() == (
uint64_t)-1)
1561 splitLargeImmediate(Opcode,
Node,
Node->getOperand(0),
1570 if (tryRxSBG(
Node, SystemZ::RNSBG))
1577 if (tryRISBGZero(
Node))
1584 if (
Node->getValueType(0) == MVT::i64) {
1585 uint64_t Val = cast<ConstantSDNode>(
Node)->getZExtValue();
1600 (Subtarget->hasLoadStoreOnCond2() &&
1601 Node->getValueType(0).isInteger() &&
1603 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1605 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1609 cast<ConstantSDNode>(CCValid.
getNode())->getZExtValue();
1611 cast<ConstantSDNode>(CCMask.
getNode())->getZExtValue();
1613 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1617 CurDAG->UpdateNodeOperands(
Node, Op1, Op0, CCValid, CCMask, Op4);
1618 if (UpdatedNode !=
Node) {
1620 ReplaceNode(
Node, UpdatedNode);
1628 EVT VT =
Node->getValueType(0);
1630 if (ElemBitSize == 32) {
1631 if (tryGather(
Node, SystemZ::VGEF))
1633 }
else if (ElemBitSize == 64) {
1634 if (tryGather(
Node, SystemZ::VGEG))
1641 auto *BVN = cast<BuildVectorSDNode>(
Node);
1644 loadVectorConstant(VCI,
Node);
1652 if (
Imm.isZero() ||
Imm.isNegZero())
1657 loadVectorConstant(VCI,
Node);
1662 if (tryFoldLoadStoreIntoMemOperand(
Node))
1665 unsigned ElemBitSize =
Store->getValue().getValueSizeInBits();
1666 if (ElemBitSize == 32) {
1667 if (tryScatter(Store, SystemZ::VSCEF))
1669 }
else if (ElemBitSize == 64) {
1670 if (tryScatter(Store, SystemZ::VSCEG))
1680bool SystemZDAGToDAGISel::
1681SelectInlineAsmMemoryOperand(
const SDValue &Op,
1682 unsigned ConstraintID,
1683 std::vector<SDValue> &OutOps) {
1684 SystemZAddressingMode::AddrForm
Form;
1685 SystemZAddressingMode::DispRange DispRange;
1688 switch(ConstraintID) {
1695 Form = SystemZAddressingMode::FormBD;
1696 DispRange = SystemZAddressingMode::Disp12Only;
1701 Form = SystemZAddressingMode::FormBDXNormal;
1702 DispRange = SystemZAddressingMode::Disp12Only;
1707 Form = SystemZAddressingMode::FormBD;
1708 DispRange = SystemZAddressingMode::Disp20Only;
1719 Form = SystemZAddressingMode::FormBDXNormal;
1720 DispRange = SystemZAddressingMode::Disp20Only;
1724 if (selectBDXAddr(
Form, DispRange, Op,
Base, Disp,
Index)) {
1726 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1728 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(),
DL, MVT::i32);
1735 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1743 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1748 OutOps.push_back(
Base);
1749 OutOps.push_back(Disp);
1750 OutOps.push_back(
Index);
1765 if (!
N.hasOneUse() || !
U->hasOneUse())
1772 SDNode *CCUser = *
U->use_begin();
1773 SDNode *CCRegUser =
nullptr;
1775 cast<RegisterSDNode>(CCUser->
getOperand(1))->getReg() == SystemZ::CC) {
1776 for (
auto *U : CCUser->
uses()) {
1777 if (CCRegUser ==
nullptr)
1779 else if (CCRegUser != U)
1783 if (CCRegUser ==
nullptr)
1796 if (!IsLegalToFold(
N, U, CCRegUser, OptLevel,
false))
1806struct IPMConversion {
1807 IPMConversion(
unsigned xorValue, int64_t addValue,
unsigned bit)
1808 : XORValue(xorValue), AddValue(addValue),
Bit(bit) {}
1884 auto *TrueOp = dyn_cast<ConstantSDNode>(
Node->getOperand(0));
1885 auto *FalseOp = dyn_cast<ConstantSDNode>(
Node->getOperand(1));
1886 if (!TrueOp || !FalseOp)
1888 if (FalseOp->getZExtValue() != 0)
1890 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
1893 auto *CCValidOp = dyn_cast<ConstantSDNode>(
Node->getOperand(2));
1894 auto *CCMaskOp = dyn_cast<ConstantSDNode>(
Node->getOperand(3));
1895 if (!CCValidOp || !CCMaskOp)
1897 int CCValid = CCValidOp->getZExtValue();
1898 int CCMask = CCMaskOp->getZExtValue();
1907 CurDAG->getConstant(
IPM.XORValue,
DL, MVT::i32));
1911 CurDAG->getConstant(
IPM.AddValue,
DL, MVT::i32));
1913 EVT VT =
Node->getValueType(0);
1914 if (VT == MVT::i32 &&
IPM.Bit == 31) {
1916 Result = CurDAG->getNode(ShiftOp,
DL, MVT::i32, Result,
1917 CurDAG->getConstant(
IPM.Bit,
DL, MVT::i32));
1922 if (TrueOp->getSExtValue() == 1) {
1925 CurDAG->getConstant(
IPM.Bit,
DL, MVT::i32));
1927 CurDAG->getConstant(1,
DL, VT));
1933 CurDAG->getConstant(ShlAmt,
DL, MVT::i32));
1935 CurDAG->getConstant(SraAmt,
DL, MVT::i32));
1942void SystemZDAGToDAGISel::PreprocessISelDAG() {
1945 if (Subtarget->hasLoadStoreOnCond2())
1948 bool MadeChange =
false;
1951 E = CurDAG->allnodes_end();
1958 switch (
N->getOpcode()) {
1961 Res = expandSelectBoolean(
N);
1966 LLVM_DEBUG(
dbgs() <<
"SystemZ DAG preprocessing replacing:\nOld: ");
1972 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1978 CurDAG->RemoveDeadNodes();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
const HexagonInstrInfo * TII
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask)
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
static uint64_t allOnes(unsigned int Count)
DEMANGLE_DUMP_METHOD void dump() const
uint64_t getZExtValue() const
Get zero extended value.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
Module * getParent()
Get the module that this global value is contained inside of...
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
An SDNode that represents everything that will be needed to construct a MachineInstr.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
bool isNonTemporal() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Special value supplied for machine level alias analysis.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
static int getUninvalidatedNodeId(SDNode *N)
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SystemZInstrInfo * getInstrInfo() const override
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
LLVM Value Representation.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
Iterator for intrusive lists based on ilist_node.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Level
Code generation optimization level.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SHL
Shift and rotation operations.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isPCREL(unsigned Opcode)
static bool isImmHF(uint64_t Val)
static bool isImmLF(uint64_t Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Or
Bitwise or logical OR of integers.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)