41 #define DEBUG_TYPE "x86-isel"
43 STATISTIC(NumLoadMoved,
"Number of loads moved below TokenFactor");
52 struct X86ISelAddressMode {
73 unsigned char SymbolFlags;
76 : BaseType(RegBase), Base_FrameIndex(0), Scale(1), IndexReg(), Disp(0),
77 Segment(), GV(nullptr),
CP(nullptr), BlockAddr(nullptr),
ES(nullptr),
78 MCSym(nullptr),
JT(-1), Align(0), SymbolFlags(X86II::
MO_NO_FLAG) {}
80 bool hasSymbolicDisplacement()
const {
81 return GV !=
nullptr ||
CP !=
nullptr ||
ES !=
nullptr ||
82 MCSym !=
nullptr ||
JT != -1 || BlockAddr !=
nullptr;
85 bool hasBaseOrIndexReg()
const {
86 return BaseType == FrameIndexBase ||
87 IndexReg.getNode() !=
nullptr || Base_Reg.getNode() !=
nullptr;
91 bool isRIPRelative()
const {
92 if (BaseType != RegBase)
return false;
94 dyn_cast_or_null<RegisterSDNode>(Base_Reg.getNode()))
95 return RegNode->getReg() == X86::RIP;
104 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
106 dbgs() <<
"X86ISelAddressMode " <<
this <<
'\n';
107 dbgs() <<
"Base_Reg ";
108 if (Base_Reg.getNode())
109 Base_Reg.getNode()->dump();
112 dbgs() <<
" Base.FrameIndex " << Base_FrameIndex <<
'\n'
113 <<
" Scale" << Scale <<
'\n'
115 if (IndexReg.getNode())
116 IndexReg.getNode()->dump();
119 dbgs() <<
" Disp " << Disp <<
'\n'
141 dbgs() <<
" JT" <<
JT <<
" Align" << Align <<
'\n';
167 OptForMinSize(
false) {}
170 return "X86 DAG->DAG Instruction Selection";
180 void EmitFunctionEntryCode()
override;
184 void PreprocessISelDAG()
override;
187 #include "X86GenDAGISel.inc"
191 bool tryGather(
SDNode *
N,
unsigned Opc);
193 bool foldOffsetIntoAddress(uint64_t
Offset, X86ISelAddressMode &AM);
194 bool matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM);
195 bool matchWrapper(
SDValue N, X86ISelAddressMode &AM);
196 bool matchAddress(
SDValue N, X86ISelAddressMode &AM);
197 bool matchAdd(
SDValue N, X86ISelAddressMode &AM,
unsigned Depth);
198 bool matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
200 bool matchAddressBase(
SDValue N, X86ISelAddressMode &AM);
230 bool SelectInlineAsmMemoryOperand(
const SDValue &
Op,
231 unsigned ConstraintID,
232 std::vector<SDValue> &OutOps)
override;
234 void emitSpecialCodeForMain();
236 inline void getAddressOperands(X86ISelAddressMode &AM,
const SDLoc &DL,
240 Base = (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
241 ? CurDAG->getTargetFrameIndex(
243 TLI->getPointerTy(CurDAG->getDataLayout()))
245 Scale = getI8Imm(AM.Scale, DL);
250 Disp = CurDAG->getTargetGlobalAddress(AM.GV,
SDLoc(),
254 Disp = CurDAG->getTargetConstantPool(AM.CP,
MVT::i32,
255 AM.Align, AM.Disp, AM.SymbolFlags);
257 assert(!AM.Disp &&
"Non-zero displacement is ignored with ES.");
258 Disp = CurDAG->getTargetExternalSymbol(AM.ES,
MVT::i32, AM.SymbolFlags);
259 }
else if (AM.MCSym) {
260 assert(!AM.Disp &&
"Non-zero displacement is ignored with MCSym.");
261 assert(AM.SymbolFlags == 0 &&
"oo");
262 Disp = CurDAG->getMCSymbol(AM.MCSym,
MVT::i32);
263 }
else if (AM.JT != -1) {
264 assert(!AM.Disp &&
"Non-zero displacement is ignored with JT.");
265 Disp = CurDAG->getTargetJumpTable(AM.JT,
MVT::i32, AM.SymbolFlags);
266 }
else if (AM.BlockAddr)
267 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr,
MVT::i32, AM.Disp,
270 Disp = CurDAG->getTargetConstant(AM.Disp, DL,
MVT::i32);
272 if (AM.Segment.getNode())
273 Segment = AM.Segment;
275 Segment = CurDAG->getRegister(0,
MVT::i32);
284 bool shouldAvoidImmediateInstFormsForSize(
SDNode *
N)
const {
295 UE = N->
use_end(); (UI != UE) && (UseCount < 2); ++UI) {
339 (RegNode = dyn_cast_or_null<RegisterSDNode>(
341 if ((RegNode->getReg() == X86::ESP) ||
342 (RegNode->getReg() == X86::RSP))
351 return (UseCount > 1);
355 inline SDValue getI8Imm(
unsigned Imm,
const SDLoc &DL) {
356 return CurDAG->getTargetConstant(Imm, DL,
MVT::i8);
360 inline SDValue getI32Imm(
unsigned Imm,
const SDLoc &DL) {
361 return CurDAG->getTargetConstant(Imm, DL,
MVT::i32);
367 SDNode *getGlobalBaseReg();
378 return Subtarget->getInstrInfo();
384 bool ComplexPatternFuncMutatesDAG()
const override {
429 if (Imm->getAPIntValue().isSignedIntN(8))
464 "Unexpected chain operand");
516 if (isa<MemSDNode>(Chain.
getNode()) &&
517 cast<MemSDNode>(Chain.
getNode())->writeMem())
528 void X86DAGToDAGISel::PreprocessISelDAG() {
530 OptForSize = MF->getFunction()->optForSize();
531 OptForMinSize = MF->getFunction()->optForMinSize();
532 assert((!OptForMinSize || OptForSize) &&
"OptForMinSize implies OptForSize");
535 E = CurDAG->allnodes_end();
I !=
E; ) {
544 (Subtarget->is64Bit() ||
545 !getTargetMachine().isPositionIndependent())))) {
599 if (SrcIsSSE && DstIsSSE)
602 if (!SrcIsSSE && !DstIsSSE) {
618 MemVT = SrcIsSSE ? SrcVT : DstVT;
620 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
625 CurDAG->getTruncStore(CurDAG->getEntryNode(), dl, N->
getOperand(0),
635 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(N, 0), Result);
640 CurDAG->DeleteNode(N);
646 void X86DAGToDAGISel::emitSpecialCodeForMain() {
647 if (Subtarget->isTargetCygMing()) {
649 auto &DL = CurDAG->getDataLayout();
652 CLI.setChain(CurDAG->getRoot())
654 CurDAG->getExternalSymbol(
"__main", TLI->getPointerTy(DL)),
657 std::pair<SDValue, SDValue> Result = TLI.
LowerCallTo(CLI);
658 CurDAG->setRoot(Result.second);
662 void X86DAGToDAGISel::EmitFunctionEntryCode() {
664 if (
const Function *Fn = MF->getFunction())
665 if (Fn->hasExternalLinkage() && Fn->getName() ==
"main")
666 emitSpecialCodeForMain();
676 return isInt<31>(Val);
679 bool X86DAGToDAGISel::foldOffsetIntoAddress(uint64_t
Offset,
680 X86ISelAddressMode &AM) {
682 if (Offset != 0 && (AM.ES || AM.MCSym))
684 int64_t Val = AM.Disp +
Offset;
686 if (Subtarget->is64Bit()) {
688 AM.hasSymbolicDisplacement()))
692 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
701 bool X86DAGToDAGISel::matchLoadInAddress(
LoadSDNode *N, X86ISelAddressMode &AM){
711 if (
C->getSExtValue() == 0 && AM.Segment.getNode() ==
nullptr &&
712 Subtarget->isTargetGlibc())
715 AM.Segment = CurDAG->getRegister(X86::GS,
MVT::i16);
718 AM.Segment = CurDAG->getRegister(X86::FS,
MVT::i16);
730 bool X86DAGToDAGISel::matchWrapper(
SDValue N, X86ISelAddressMode &AM) {
733 if (AM.hasSymbolicDisplacement())
747 if (AM.hasBaseOrIndexReg())
750 X86ISelAddressMode Backup = AM;
751 AM.GV =
G->getGlobal();
752 AM.SymbolFlags =
G->getTargetFlags();
753 if (foldOffsetIntoAddress(
G->getOffset(), AM)) {
758 X86ISelAddressMode Backup = AM;
759 AM.CP =
CP->getConstVal();
760 AM.Align =
CP->getAlignment();
761 AM.SymbolFlags =
CP->getTargetFlags();
762 if (foldOffsetIntoAddress(
CP->getOffset(), AM)) {
767 AM.ES = S->getSymbol();
768 AM.SymbolFlags = S->getTargetFlags();
769 }
else if (
auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
770 AM.MCSym = S->getMCSymbol();
772 AM.JT = J->getIndex();
773 AM.SymbolFlags = J->getTargetFlags();
775 X86ISelAddressMode Backup = AM;
776 AM.BlockAddr = BA->getBlockAddress();
777 AM.SymbolFlags = BA->getTargetFlags();
778 if (foldOffsetIntoAddress(BA->getOffset(), AM)) {
786 AM.setBaseReg(CurDAG->getRegister(X86::RIP,
MVT::i64));
793 if (!Subtarget->is64Bit() ||
796 "RIP-relative addressing already handled");
798 AM.GV =
G->getGlobal();
799 AM.Disp +=
G->getOffset();
800 AM.SymbolFlags =
G->getTargetFlags();
802 AM.CP =
CP->getConstVal();
803 AM.Align =
CP->getAlignment();
804 AM.Disp +=
CP->getOffset();
805 AM.SymbolFlags =
CP->getTargetFlags();
807 AM.ES = S->getSymbol();
808 AM.SymbolFlags = S->getTargetFlags();
809 }
else if (
auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
810 AM.MCSym = S->getMCSymbol();
812 AM.JT = J->getIndex();
813 AM.SymbolFlags = J->getTargetFlags();
815 AM.BlockAddr = BA->getBlockAddress();
816 AM.Disp += BA->getOffset();
817 AM.SymbolFlags = BA->getTargetFlags();
828 bool X86DAGToDAGISel::matchAddress(
SDValue N, X86ISelAddressMode &AM) {
829 if (matchAddressRecursively(N, AM, 0))
835 AM.BaseType == X86ISelAddressMode::RegBase &&
836 AM.Base_Reg.getNode() ==
nullptr) {
837 AM.Base_Reg = AM.IndexReg;
845 Subtarget->is64Bit() &&
847 AM.BaseType == X86ISelAddressMode::RegBase &&
848 AM.Base_Reg.getNode() ==
nullptr &&
849 AM.IndexReg.getNode() ==
nullptr &&
851 AM.hasSymbolicDisplacement())
852 AM.Base_Reg = CurDAG->getRegister(X86::RIP,
MVT::i64);
857 bool X86DAGToDAGISel::matchAdd(
SDValue N, X86ISelAddressMode &AM,
863 X86ISelAddressMode Backup = AM;
864 if (!matchAddressRecursively(N.
getOperand(0), AM, Depth+1) &&
865 !matchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1))
870 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1) &&
871 !matchAddressRecursively(Handle.getValue().getOperand(0), AM, Depth+1))
878 if (AM.BaseType == X86ISelAddressMode::RegBase &&
879 !AM.Base_Reg.getNode() &&
880 !AM.IndexReg.getNode()) {
911 X86ISelAddressMode &AM) {
918 if (ScaleLog <= 0 || ScaleLog >= 4 ||
919 Mask != (0xffu << ScaleLog))
944 AM.Scale = (1 << ScaleLog);
954 X86ISelAddressMode &AM) {
967 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3)
986 AM.Scale = 1 << ShiftAmt;
987 AM.IndexReg = NewAnd;
1021 X86ISelAddressMode &AM) {
1032 unsigned AMShiftAmt = MaskTZ;
1036 if (AMShiftAmt <= 0 || AMShiftAmt > 3)
return true;
1051 bool ReplacingAnyExtend =
false;
1058 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits;
1059 ReplacingAnyExtend =
true;
1061 APInt MaskedHighBits =
1063 APInt KnownZero, KnownOne;
1065 if (MaskedHighBits != KnownZero)
return true;
1070 if (ReplacingAnyExtend) {
1094 AM.Scale = 1 << AMShiftAmt;
1095 AM.IndexReg = NewSRL;
1099 bool X86DAGToDAGISel::matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
1103 dbgs() <<
"MatchAddress: ";
1108 return matchAddressBase(N, AM);
1113 if (AM.isRIPRelative()) {
1117 if (!(AM.ES || AM.MCSym) && AM.JT != -1)
1121 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM))
1129 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
1130 if (
const auto *ESNode = dyn_cast<MCSymbolSDNode>(N.
getOperand(0))) {
1132 AM.MCSym = ESNode->getMCSymbol();
1138 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
1139 if (!foldOffsetIntoAddress(Val, AM))
1146 if (!matchWrapper(N, AM))
1151 if (!matchLoadInAddress(cast<LoadSDNode>(N), AM))
1156 if (AM.BaseType == X86ISelAddressMode::RegBase &&
1157 AM.Base_Reg.getNode() ==
nullptr &&
1159 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
1160 AM.Base_FrameIndex = cast<FrameIndexSDNode>(
N)->getIndex();
1166 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
1171 unsigned Val = CN->getZExtValue();
1176 if (Val == 1 || Val == 2 || Val == 3) {
1177 AM.Scale = 1 << Val;
1183 if (CurDAG->isBaseWithConstantOffset(ShVal)) {
1187 uint64_t Disp = (uint64_t)AddVal->
getSExtValue() << Val;
1188 if (!foldOffsetIntoAddress(Disp, AM))
1192 AM.IndexReg = ShVal;
1200 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
1233 if (AM.BaseType == X86ISelAddressMode::RegBase &&
1234 AM.Base_Reg.getNode() ==
nullptr &&
1235 AM.IndexReg.getNode() ==
nullptr) {
1238 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
1239 CN->getZExtValue() == 9) {
1240 AM.Scale =
unsigned(CN->getZExtValue())-1;
1248 if (MulVal.getNode()->getOpcode() ==
ISD::ADD && MulVal.hasOneUse() &&
1249 isa<ConstantSDNode>(MulVal.getNode()->getOperand(1))) {
1252 cast<ConstantSDNode>(MulVal.getNode()->getOperand(1));
1253 uint64_t Disp = AddVal->
getSExtValue() * CN->getZExtValue();
1254 if (foldOffsetIntoAddress(Disp, AM))
1260 AM.IndexReg = AM.Base_Reg =
Reg;
1279 X86ISelAddressMode Backup = AM;
1285 if (AM.IndexReg.getNode() || AM.isRIPRelative()) {
1304 if ((AM.BaseType == X86ISelAddressMode::RegBase &&
1305 AM.Base_Reg.getNode() &&
1306 !AM.Base_Reg.getNode()->hasOneUse()) ||
1307 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
1311 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
1312 ((AM.Disp != 0) && (Backup.Disp == 0)) +
1313 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
1334 if (!matchAdd(N, AM, Depth))
1346 !matchAdd(N, AM, Depth))
1355 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
1385 return matchAddressBase(N, AM);
1390 bool X86DAGToDAGISel::matchAddressBase(
SDValue N, X86ISelAddressMode &AM) {
1392 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
1394 if (!AM.IndexReg.getNode()) {
1405 AM.BaseType = X86ISelAddressMode::RegBase;
1417 X86ISelAddressMode AM;
1420 if (AddrSpace == 256)
1421 AM.Segment = CurDAG->getRegister(X86::GS,
MVT::i16);
1422 if (AddrSpace == 257)
1423 AM.Segment = CurDAG->getRegister(X86::FS,
MVT::i16);
1424 if (AddrSpace == 258)
1425 AM.Segment = CurDAG->getRegister(X86::SS,
MVT::i16);
1431 Scale = getI8Imm(ScalarSize/8, DL);
1434 if (isa<ConstantSDNode>(Base)) {
1435 assert(cast<ConstantSDNode>(Base)->isNullValue() &&
1436 "Unexpected base in gather/scatter");
1437 Scale = getI8Imm(1, DL);
1438 Base = CurDAG->getRegister(0,
MVT::i32);
1440 if (AM.Segment.getNode())
1441 Segment = AM.Segment;
1443 Segment = CurDAG->getRegister(0,
MVT::i32);
1444 Disp = CurDAG->getTargetConstant(0, DL,
MVT::i32);
1458 X86ISelAddressMode AM;
1468 unsigned AddrSpace =
1469 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
1471 if (AddrSpace == 256)
1472 AM.Segment = CurDAG->getRegister(X86::GS,
MVT::i16);
1473 if (AddrSpace == 257)
1474 AM.Segment = CurDAG->getRegister(X86::FS,
MVT::i16);
1475 if (AddrSpace == 258)
1476 AM.Segment = CurDAG->getRegister(X86::SS,
MVT::i16);
1479 if (matchAddress(N, AM))
1483 if (AM.BaseType == X86ISelAddressMode::RegBase) {
1484 if (!AM.Base_Reg.getNode())
1485 AM.Base_Reg = CurDAG->getRegister(0, VT);
1488 if (!AM.IndexReg.getNode())
1489 AM.IndexReg = CurDAG->getRegister(0, VT);
1491 getAddressOperands(AM,
SDLoc(N), Base, Scale, Index, Disp, Segment);
1502 bool X86DAGToDAGISel::selectScalarSSELoad(
SDNode *Root,
1506 SDValue &PatternNodeWithChain) {
1509 PatternNodeWithChain =
N;
1510 if (IsProfitableToFold(PatternNodeWithChain, N.
getNode(), Root) &&
1511 IsLegalToFold(PatternNodeWithChain, *N->
use_begin(), Root, OptLevel)) {
1512 LoadSDNode *
LD = cast<LoadSDNode>(PatternNodeWithChain);
1513 return selectAddr(LD, LD->
getBasePtr(), Base, Scale, Index, Disp,
1520 PatternNodeWithChain =
N;
1521 if (IsProfitableToFold(PatternNodeWithChain, N.
getNode(), Root) &&
1522 IsLegalToFold(PatternNodeWithChain, *N->
use_begin(), Root, OptLevel)) {
1523 auto *
MI = cast<MemIntrinsicSDNode>(PatternNodeWithChain);
1524 return selectAddr(
MI,
MI->getBasePtr(), Base, Scale, Index, Disp,
1535 IsProfitableToFold(PatternNodeWithChain, N.
getNode(), Root) &&
1536 IsLegalToFold(PatternNodeWithChain, N.
getNode(), Root, OptLevel)) {
1537 LoadSDNode *
LD = cast<LoadSDNode>(PatternNodeWithChain);
1538 return selectAddr(LD, LD->
getBasePtr(), Base, Scale, Index, Disp,
1551 IsProfitableToFold(PatternNodeWithChain, N.
getNode(), Root) &&
1552 IsLegalToFold(PatternNodeWithChain, N.
getNode(), Root, OptLevel)) {
1554 LoadSDNode *
LD = cast<LoadSDNode>(PatternNodeWithChain);
1555 return selectAddr(LD, LD->
getBasePtr(), Base, Scale, Index, Disp,
1564 bool X86DAGToDAGISel::selectMOV64Imm32(
SDValue N,
SDValue &Imm) {
1566 uint64_t ImmVal = CN->getZExtValue();
1567 if ((
uint32_t)ImmVal != (uint64_t)ImmVal)
1570 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(N),
MVT::i64);
1578 "Unexpected node type for MOV32ri64");
1591 cast<GlobalAddressSDNode>(
N)->getGlobal()->getAbsoluteSymbolRange();
1598 bool X86DAGToDAGISel::selectLEA64_32Addr(
SDValue N,
SDValue &Base,
1604 if (!selectLEAAddr(N, Base, Scale, Index, Disp, Segment))
1608 if (RN && RN->
getReg() == 0)
1609 Base = CurDAG->getRegister(0,
MVT::i64);
1612 Base =
SDValue(CurDAG->getMachineNode(
1613 TargetOpcode::SUBREG_TO_REG, DL,
MVT::i64,
1614 CurDAG->getTargetConstant(0, DL,
MVT::i64),
1616 CurDAG->getTargetConstant(X86::sub_32bit, DL,
MVT::i32)),
1621 if (RN && RN->
getReg() == 0)
1622 Index = CurDAG->getRegister(0,
MVT::i64);
1625 "Expect to be extending 32-bit registers for use in LEA");
1626 Index =
SDValue(CurDAG->getMachineNode(
1627 TargetOpcode::SUBREG_TO_REG, DL,
MVT::i64,
1628 CurDAG->getTargetConstant(0, DL,
MVT::i64),
1630 CurDAG->getTargetConstant(X86::sub_32bit, DL,
1640 bool X86DAGToDAGISel::selectLEAAddr(
SDValue N,
1644 X86ISelAddressMode AM;
1655 if (matchAddress(N, AM))
1657 assert (T == AM.Segment);
1660 unsigned Complexity = 0;
1661 if (AM.BaseType == X86ISelAddressMode::RegBase)
1662 if (AM.Base_Reg.getNode())
1665 AM.Base_Reg = CurDAG->getRegister(0, VT);
1666 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
1669 if (AM.IndexReg.getNode())
1672 AM.IndexReg = CurDAG->getRegister(0, VT);
1684 if (AM.hasSymbolicDisplacement()) {
1686 if (Subtarget->is64Bit())
1692 if (AM.Disp && (AM.Base_Reg.getNode() || AM.IndexReg.getNode()))
1696 if (Complexity <= 2)
1699 getAddressOperands(AM, DL, Base, Scale, Index, Disp, Segment);
1704 bool X86DAGToDAGISel::selectTLSADDRAddr(
SDValue N,
SDValue &Base,
1710 X86ISelAddressMode AM;
1711 AM.GV = GA->getGlobal();
1712 AM.Disp += GA->getOffset();
1713 AM.Base_Reg = CurDAG->getRegister(0, N.
getValueType());
1714 AM.SymbolFlags = GA->getTargetFlags();
1718 AM.IndexReg = CurDAG->getRegister(X86::EBX,
MVT::i32);
1720 AM.IndexReg = CurDAG->getRegister(0,
MVT::i64);
1723 getAddressOperands(AM,
SDLoc(N), Base, Scale, Index, Disp, Segment);
1728 if (
auto *CN = dyn_cast<ConstantSDNode>(N)) {
1729 Op = CurDAG->getTargetConstant(CN->getAPIntValue(),
SDLoc(CN),
1738 bool WasTruncated =
false;
1740 WasTruncated =
true;
1755 return !WasTruncated;
1759 auto *GA = cast<GlobalAddressSDNode>(N.
getOperand(0));
1761 if (!CR || CR->getUnsignedMax().uge(1ull << VT.
getSizeInBits()))
1765 Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(),
SDLoc(N), VT,
1766 GA->getOffset(), GA->getTargetFlags());
1775 !IsProfitableToFold(N, P, P) ||
1776 !IsLegalToFold(N, P, P, OptLevel))
1779 return selectAddr(N.
getNode(),
1780 N.
getOperand(1), Base, Scale, Index, Disp, Segment);
1786 SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
1787 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
1788 auto &DL = MF->getDataLayout();
1789 return CurDAG->getRegister(GlobalBaseReg, TLI->
getPointerTy(DL)).getNode();
1797 UE = N->
use_end(); UI != UE; ++UI) {
1802 if (cast<RegisterSDNode>(UI->getOperand(1))->
getReg() !=
1807 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
1809 if (FlagUI.getUse().getResNo() != 1)
continue;
1811 if (!FlagUI->isMachineOpcode())
return false;
1813 switch (FlagUI->getMachineOpcode()) {
1815 case X86::SETAr:
case X86::SETAEr:
case X86::SETBr:
case X86::SETBEr:
1816 case X86::SETEr:
case X86::SETNEr:
case X86::SETPr:
case X86::SETNPr:
1817 case X86::SETAm:
case X86::SETAEm:
case X86::SETBm:
case X86::SETBEm:
1818 case X86::SETEm:
case X86::SETNEm:
case X86::SETPm:
case X86::SETNPm:
1819 case X86::JA_1:
case X86::JAE_1:
case X86::JB_1:
case X86::JBE_1:
1820 case X86::JE_1:
case X86::JNE_1:
case X86::JP_1:
case X86::JNP_1:
1821 case X86::CMOVA16rr:
case X86::CMOVA16rm:
1822 case X86::CMOVA32rr:
case X86::CMOVA32rm:
1823 case X86::CMOVA64rr:
case X86::CMOVA64rm:
1824 case X86::CMOVAE16rr:
case X86::CMOVAE16rm:
1825 case X86::CMOVAE32rr:
case X86::CMOVAE32rm:
1826 case X86::CMOVAE64rr:
case X86::CMOVAE64rm:
1827 case X86::CMOVB16rr:
case X86::CMOVB16rm:
1828 case X86::CMOVB32rr:
case X86::CMOVB32rm:
1829 case X86::CMOVB64rr:
case X86::CMOVB64rm:
1830 case X86::CMOVBE16rr:
case X86::CMOVBE16rm:
1831 case X86::CMOVBE32rr:
case X86::CMOVBE32rm:
1832 case X86::CMOVBE64rr:
case X86::CMOVBE64rm:
1833 case X86::CMOVE16rr:
case X86::CMOVE16rm:
1834 case X86::CMOVE32rr:
case X86::CMOVE32rm:
1835 case X86::CMOVE64rr:
case X86::CMOVE64rm:
1836 case X86::CMOVNE16rr:
case X86::CMOVNE16rm:
1837 case X86::CMOVNE32rr:
case X86::CMOVNE32rm:
1838 case X86::CMOVNE64rr:
case X86::CMOVNE64rm:
1839 case X86::CMOVNP16rr:
case X86::CMOVNP16rm:
1840 case X86::CMOVNP32rr:
case X86::CMOVNP32rm:
1841 case X86::CMOVNP64rr:
case X86::CMOVNP64rm:
1842 case X86::CMOVP16rr:
case X86::CMOVP16rm:
1843 case X86::CMOVP32rr:
case X86::CMOVP32rm:
1844 case X86::CMOVP64rr:
case X86::CMOVP64rm:
1847 default:
return false;
1864 if (StoredVal.
getResNo() != 0)
return false;
1878 LoadNode = cast<LoadSDNode>(
Load);
1898 bool ChainCheck =
false;
1917 UE = UI->
use_end(); UI != UE; ++UI) {
1918 if (UI.getUse().getResNo() != 0)
1920 if (UI->getNodeId() > LoadId)
1943 if (LdVT ==
MVT::i64)
return X86::DEC64m;
1944 if (LdVT ==
MVT::i32)
return X86::DEC32m;
1945 if (LdVT ==
MVT::i16)
return X86::DEC16m;
1946 if (LdVT ==
MVT::i8)
return X86::DEC8m;
1949 if (LdVT ==
MVT::i64)
return X86::INC64m;
1950 if (LdVT ==
MVT::i32)
return X86::INC32m;
1951 if (LdVT ==
MVT::i16)
return X86::INC16m;
1952 if (LdVT ==
MVT::i8)
return X86::INC8m;
1958 bool X86DAGToDAGISel::tryGather(
SDNode *Node,
unsigned Opc) {
1978 Disp, Segment, VMask, Chain};
1979 SDNode *ResNode = CurDAG->getMachineNode(Opc, DL, VTs, Ops);
1986 CurDAG->RemoveDeadNode(Node);
2007 if (Subtarget->isTargetNaCl())
2011 if (Subtarget->isTarget64BitILP32()) {
2020 ReplaceNode(Node, Brind.
getNode());
2021 SelectCode(ZextTarget.getNode());
2028 unsigned IntNo = cast<ConstantSDNode>(Node->
getOperand(1))->getZExtValue();
2031 case Intrinsic::x86_avx2_gather_d_pd:
2032 case Intrinsic::x86_avx2_gather_d_pd_256:
2033 case Intrinsic::x86_avx2_gather_q_pd:
2034 case Intrinsic::x86_avx2_gather_q_pd_256:
2035 case Intrinsic::x86_avx2_gather_d_ps:
2036 case Intrinsic::x86_avx2_gather_d_ps_256:
2037 case Intrinsic::x86_avx2_gather_q_ps:
2038 case Intrinsic::x86_avx2_gather_q_ps_256:
2039 case Intrinsic::x86_avx2_gather_d_q:
2040 case Intrinsic::x86_avx2_gather_d_q_256:
2041 case Intrinsic::x86_avx2_gather_q_q:
2042 case Intrinsic::x86_avx2_gather_q_q_256:
2043 case Intrinsic::x86_avx2_gather_d_d:
2044 case Intrinsic::x86_avx2_gather_d_d_256:
2045 case Intrinsic::x86_avx2_gather_q_d:
2046 case Intrinsic::x86_avx2_gather_q_d_256: {
2047 if (!Subtarget->hasAVX2())
2052 case Intrinsic::x86_avx2_gather_d_pd: Opc = X86::VGATHERDPDrm;
break;
2053 case Intrinsic::x86_avx2_gather_d_pd_256: Opc = X86::VGATHERDPDYrm;
break;
2054 case Intrinsic::x86_avx2_gather_q_pd: Opc = X86::VGATHERQPDrm;
break;
2055 case Intrinsic::x86_avx2_gather_q_pd_256: Opc = X86::VGATHERQPDYrm;
break;
2056 case Intrinsic::x86_avx2_gather_d_ps: Opc = X86::VGATHERDPSrm;
break;
2057 case Intrinsic::x86_avx2_gather_d_ps_256: Opc = X86::VGATHERDPSYrm;
break;
2058 case Intrinsic::x86_avx2_gather_q_ps: Opc = X86::VGATHERQPSrm;
break;
2059 case Intrinsic::x86_avx2_gather_q_ps_256: Opc = X86::VGATHERQPSYrm;
break;
2060 case Intrinsic::x86_avx2_gather_d_q: Opc = X86::VPGATHERDQrm;
break;
2061 case Intrinsic::x86_avx2_gather_d_q_256: Opc = X86::VPGATHERDQYrm;
break;
2062 case Intrinsic::x86_avx2_gather_q_q: Opc = X86::VPGATHERQQrm;
break;
2063 case Intrinsic::x86_avx2_gather_q_q_256: Opc = X86::VPGATHERQQYrm;
break;
2064 case Intrinsic::x86_avx2_gather_d_d: Opc = X86::VPGATHERDDrm;
break;
2065 case Intrinsic::x86_avx2_gather_d_d_256: Opc = X86::VPGATHERDDYrm;
break;
2066 case Intrinsic::x86_avx2_gather_q_d: Opc = X86::VPGATHERQDrm;
break;
2067 case Intrinsic::x86_avx2_gather_q_d_256: Opc = X86::VPGATHERQDYrm;
break;
2069 if (tryGather(Node, Opc))
2077 ReplaceNode(Node, getGlobalBaseReg());
2082 SDValue VSelect = CurDAG->getNode(
2085 ReplaceUses(
SDValue(Node, 0), VSelect);
2086 SelectCode(VSelect.
getNode());
2108 if (!Cst || !ShlCst)
2112 uint64_t ShlVal = ShlCst->getZExtValue();
2116 uint64_t RemovedBitsMask = (1ULL << ShlVal) - 1;
2117 if (Opcode !=
ISD::AND && (Val & RemovedBitsMask) != 0)
2120 unsigned ShlOp, AddOp,
Op;
2140 ShlOp = X86::SHL32ri;
2141 AddOp = X86::ADD32rr;
2145 case ISD::AND: Op = X86::AND32ri8;
break;
2146 case ISD::OR: Op = X86::OR32ri8;
break;
2147 case ISD::XOR: Op = X86::XOR32ri8;
break;
2152 ShlOp = X86::SHL64ri;
2153 AddOp = X86::ADD64rr;
2157 case ISD::AND: Op = CstVT==
MVT::i8? X86::AND64ri8 : X86::AND64ri32;
break;
2158 case ISD::OR: Op = CstVT==
MVT::i8? X86::OR64ri8 : X86::OR64ri32;
break;
2159 case ISD::XOR: Op = CstVT==
MVT::i8? X86::XOR64ri8 : X86::XOR64ri32;
break;
2165 SDValue NewCst = CurDAG->getTargetConstant(Val >> ShlVal, dl, CstVT);
2166 SDNode *New = CurDAG->getMachineNode(Op, dl, NVT, N0->
getOperand(0),NewCst);
2168 CurDAG->SelectNodeTo(Node, AddOp, NVT,
SDValue(New, 0),
2171 CurDAG->SelectNodeTo(Node, ShlOp, NVT,
SDValue(New, 0),
2172 getI8Imm(ShlVal, dl));
2182 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::AL,
2187 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
2189 ReplaceNode(Node, CNode);
2200 case MVT::i8: LoReg = X86::AL; Opc = X86::MUL8r;
break;
2201 case MVT::i16: LoReg = X86::AX; Opc = X86::MUL16r;
break;
2202 case MVT::i32: LoReg = X86::EAX; Opc = X86::MUL32r;
break;
2203 case MVT::i64: LoReg = X86::RAX; Opc = X86::MUL64r;
break;
2206 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
2211 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
2213 ReplaceNode(Node, CNode);
2223 bool hasBMI2 = Subtarget->hasBMI2();
2227 case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m;
break;
2228 case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m;
break;
2229 case MVT::i32: Opc = hasBMI2 ? X86::MULX32rr : X86::MUL32r;
2230 MOpc = hasBMI2 ? X86::MULX32rm : X86::MUL32m;
break;
2231 case MVT::i64: Opc = hasBMI2 ? X86::MULX64rr : X86::MUL64r;
2232 MOpc = hasBMI2 ? X86::MULX64rm : X86::MUL64m;
break;
2237 case MVT::i8: Opc = X86::IMUL8r; MOpc = X86::IMUL8m;
break;
2238 case MVT::i16: Opc = X86::IMUL16r; MOpc = X86::IMUL16m;
break;
2239 case MVT::i32: Opc = X86::IMUL32r; MOpc = X86::IMUL32m;
break;
2240 case MVT::i64: Opc = X86::IMUL64r; MOpc = X86::IMUL64m;
break;
2244 unsigned SrcReg, LoReg, HiReg;
2249 SrcReg = LoReg = X86::AL; HiReg = X86::AH;
2253 SrcReg = LoReg = X86::AX; HiReg = X86::DX;
2257 SrcReg = LoReg = X86::EAX; HiReg = X86::EDX;
2261 SrcReg = LoReg = X86::RAX; HiReg = X86::RDX;
2264 SrcReg = X86::EDX; LoReg = HiReg = 0;
2267 SrcReg = X86::RDX; LoReg = HiReg = 0;
2271 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
2272 bool foldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
2275 foldedLoad = tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
2280 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, SrcReg,
2289 if (MOpc == X86::MULX32rm || MOpc == X86::MULX64rm) {
2291 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
2298 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
2304 ReplaceUses(N1.
getValue(1), Chain);
2309 MemOp[0] = LoadNode->getMemOperand();
2313 SDValue Ops[] = { N1, InFlag };
2314 if (Opc == X86::MULX32rr || Opc == X86::MULX64rr) {
2316 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
2322 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
2328 if (HiReg == X86::AH && Subtarget->is64Bit() &&
2330 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
2335 if (!
SDValue(Node, 0).use_empty())
2337 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl,
MVT::i8, Result));
2340 Result =
SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16,
2342 CurDAG->getTargetConstant(8, dl,
MVT::i8)),
2346 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl,
MVT::i8, Result));
2351 assert(LoReg &&
"Register for low half is not defined!");
2352 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg, NVT,
2356 ReplaceUses(
SDValue(Node, 0), ResLo);
2362 assert(HiReg &&
"Register for high half is not defined!");
2363 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg, NVT,
2367 ReplaceUses(
SDValue(Node, 1), ResHi);
2384 switch (NVT.SimpleTy) {
2386 case MVT::i8: Opc = X86::DIV8r; MOpc = X86::DIV8m;
break;
2387 case MVT::i16: Opc = X86::DIV16r; MOpc = X86::DIV16m;
break;
2388 case MVT::i32: Opc = X86::DIV32r; MOpc = X86::DIV32m;
break;
2389 case MVT::i64: Opc = X86::DIV64r; MOpc = X86::DIV64m;
break;
2392 switch (NVT.SimpleTy) {
2394 case MVT::i8: Opc = X86::IDIV8r; MOpc = X86::IDIV8m;
break;
2395 case MVT::i16: Opc = X86::IDIV16r; MOpc = X86::IDIV16m;
break;
2396 case MVT::i32: Opc = X86::IDIV32r; MOpc = X86::IDIV32m;
break;
2397 case MVT::i64: Opc = X86::IDIV64r; MOpc = X86::IDIV64m;
break;
2401 unsigned LoReg, HiReg, ClrReg;
2402 unsigned SExtOpcode;
2403 switch (NVT.SimpleTy) {
2406 LoReg = X86::AL; ClrReg = HiReg = X86::AH;
2407 SExtOpcode = X86::CBW;
2410 LoReg = X86::AX; HiReg = X86::DX;
2412 SExtOpcode = X86::CWD;
2415 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX;
2416 SExtOpcode = X86::CDQ;
2419 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX;
2420 SExtOpcode = X86::CQO;
2424 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
2425 bool foldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
2426 bool signBitIsZero = CurDAG->SignBitIsZero(N0);
2429 if (NVT ==
MVT::i8 && (!isSigned || signBitIsZero)) {
2432 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Move, Chain;
2433 if (tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
2439 ReplaceUses(N0.
getValue(1), Chain);
2443 Chain = CurDAG->getEntryNode();
2445 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EAX, Move,
SDValue());
2449 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
2450 LoReg, N0,
SDValue()).getValue(1);
2451 if (isSigned && !signBitIsZero) {
2457 SDValue ClrNode =
SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, NVT), 0);
2458 switch (NVT.SimpleTy) {
2461 SDValue(CurDAG->getMachineNode(
2462 TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode,
2463 CurDAG->getTargetConstant(X86::sub_16bit, dl,
2471 SDValue(CurDAG->getMachineNode(
2472 TargetOpcode::SUBREG_TO_REG, dl,
MVT::i64,
2473 CurDAG->getTargetConstant(0, dl,
MVT::i64), ClrNode,
2474 CurDAG->getTargetConstant(X86::sub_32bit, dl,
2482 InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg,
2483 ClrNode, InFlag).getValue(1);
2509 unsigned AHExtOpcode =
2510 isSigned ? X86::MOVSX32_NOREXrr8 : X86::MOVZX32_NOREXrr8;
2512 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl,
MVT::i32,
2523 "Unexpected i64 sext of h-register");
2525 SDValue(CurDAG->getMachineNode(
2526 TargetOpcode::SUBREG_TO_REG, dl,
MVT::i64,
2527 CurDAG->getTargetConstant(0, dl,
MVT::i64), Result,
2528 CurDAG->getTargetConstant(X86::sub_32bit, dl,
2534 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl,
MVT::i8, Result);
2536 ReplaceUses(
SDValue(Node, 1), Result);
2537 DEBUG(
dbgs() <<
"=> "; Result.getNode()->dump(CurDAG);
dbgs() <<
'\n');
2541 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
2542 LoReg, NVT, InFlag);
2544 ReplaceUses(
SDValue(Node, 0), Result);
2549 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
2550 HiReg, NVT, InFlag);
2552 ReplaceUses(
SDValue(Node, 1), Result);
2590 if (!Subtarget->is64Bit()) {
2593 case MVT::i32: TRC = &X86::GR32_ABCDRegClass;
break;
2594 case MVT::i16: TRC = &X86::GR16_ABCDRegClass;
break;
2598 Reg =
SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl,
2603 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl,
2607 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri, dl,
MVT::i32,
2628 case MVT::i64: TRC = &X86::GR64_ABCDRegClass;
break;
2629 case MVT::i32: TRC = &X86::GR32_ABCDRegClass;
break;
2630 case MVT::i16: TRC = &X86::GR16_ABCDRegClass;
break;
2634 Reg =
SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl,
2638 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl,
2644 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri_NOREX, dl,
2663 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl,
2667 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST16ri, dl,
MVT::i32,
2677 if ((C->
getZExtValue() & ~UINT64_C(0xffffffff)) == 0 &&
2686 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_32bit, dl,
2690 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST32ri, dl,
MVT::i32,
2721 unsigned Opc = StoredVal->getOpcode();
2726 LoadNode, InputChain))
2729 SDValue Base, Scale, Index, Disp, Segment;
2730 if (!selectAddr(LoadNode, LoadNode->
getBasePtr(),
2731 Base, Scale, Index, Disp, Segment))
2737 const SDValue Ops[] = { Base, Scale, Index, Disp, Segment, InputChain };
2746 ReplaceUses(
SDValue(StoredVal.getNode(), 1),
SDValue(Result, 0));
2747 CurDAG->RemoveDeadNode(Node);
2755 bool X86DAGToDAGISel::
2756 SelectInlineAsmMemoryOperand(
const SDValue &Op,
unsigned ConstraintID,
2757 std::vector<SDValue> &OutOps) {
2758 SDValue Op0, Op1, Op2, Op3, Op4;
2759 switch (ConstraintID) {
2770 if (!selectAddr(
nullptr, Op, Op0, Op1, Op2, Op3, Op4))
2775 OutOps.push_back(Op0);
2776 OutOps.push_back(Op1);
2777 OutOps.push_back(Op2);
2778 OutOps.push_back(Op3);
2779 OutOps.push_back(Op4);
2787 return new X86DAGToDAGISel(TM, OptLevel);
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq)
Return true if call address is a load and it can be moved below CALLSEQ_START and the chains leading ...
void push_back(const T &Elt)
SDValue getValue(unsigned R) const
const SDValue & getValue() const
STATISTIC(NumFunctions,"Total number of functions")
static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N)
void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd)
Assign this MachineSDNodes's memory reference descriptor list.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
Carry-setting nodes for multiple precision addition and subtraction.
void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in the KnownZero/KnownO...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getID() const
Return the register class ID number.
constexpr bool isInt< 8 >(int64_t x)
unsigned getSizeInBits() const
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getNumOperands() const
const SDValue & getOperand(unsigned Num) const
void setNodeId(int Id)
Set unique node id.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const SDValue & getBasePtr() const
std::size_t countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1...
unsigned getResNo() const
get the index which selects a specific result in the SDNode
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
GlobalBaseReg - On Darwin, this node represents the result of the mflr at function entry...
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
X86 compare and logical compare instructions.
The address of a basic block.
A description of a memory reference used in the backend.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Shift and rotation operations.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
std::size_t countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
CopyToReg - This node has three operands: a chain, a register number to set to this value...
Reg
All possible values of the reg field in the ModR/M byte.
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence, and carry arbitrary information that target might want to know.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
static bool isLoadIncOrDecStore(StoreSDNode *StoreNode, unsigned Opc, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
Check whether or not the chain ending in StoreNode is suitable for doing the {load; increment or decr...
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Simple integer binary arithmetic operators.
Function Alias Analysis false
const SDValue & getBasePtr() const
EVT getMemoryVT() const
Return the type of the in-memory value.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
This class is used to represent ISD::STORE nodes.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
SDNode * getNode() const
get the SDNode which holds the desired result
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
MVT - Machine Value Type.
const SDValue & getOperand(unsigned i) const
static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
bool isNonTemporal() const
C - The default llvm calling convention, compatible with C.
This is an important base class in LLVM.
bool isVector() const
isVector - Return true if this is a vector value type.
Carry-using nodes for multiple precision addition and subtraction.
This is a base class used to represent MGATHER and MSCATTER nodes.
unsigned getScalarValueSizeInBits() const
static Type * getVoidTy(LLVMContext &C)
This class provides iterator support for SDUse operands that use a specific SDNode.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
unsigned getOpcode() const
FunctionPass class - This class is used to implement most global optimizations.
const SDValue & getBasePtr() const
On Darwin, this node represents the result of the popl at function entry, used for PIC code...
self_iterator getIterator()
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
EVT - Extended Value Type.
std::vector< ArgListEntry > ArgListTy
This structure contains all information that is necessary for lowering calls.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
This class contains a discriminated union of information about pointers in memory operands...
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getOffset() const
These operations represent an abstract X86 call instruction, which includes a bunch of information...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Iterator for intrusive lists based on ilist_node.
constexpr bool isInt< 32 >(int64_t x)
void dump() const
Dump this node, for debugging.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
An SDNode that represents everything that will be needed to construct a MachineInstr.
const SDValue & getChain() const
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
Represents one node in the SelectionDAG.
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
Target - Wrapper for Target specific information.
Class for arbitrary precision integers.
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
int64_t getSExtValue() const
op_iterator op_begin() const
static use_iterator use_end()
ZERO_EXTEND - Used for integer types, zeroing the new bits.
ANY_EXTEND - Used for integer types. The high bits are undefined.
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
static bool isDispSafeForFrameIndex(int64_t Val)
uint64_t getConstantOperandVal(unsigned i) const
static unsigned getFusedLdStOpcode(EVT &LdVT, unsigned Opc)
Get the appropriate X86 opcode for an in-memory increment or decrement.
Bitwise operators - logical and, logical or, logical xor.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress, TargetGlobalTLSAddress, MCSymbol and TargetBlockAddress.
FunctionPass * createX86ISelDag(X86TargetMachine &TM, CodeGenOpt::Level OptLevel)
This pass converts a legalized DAG into a X86-specific DAG, ready for instruction scheduling...
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
op_iterator op_end() const
const SDValue & getOffset() const
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
const SDValue & getIndex() const
int getNodeId() const
Return the unique node id.
static bool hasNoSignedComparisonUses(SDNode *N)
Test whether the given X86ISD::CMP node has any uses which require the SF or OF bits to be accurate...
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
This class is used to form a handle around another node that is persistent and is updated across invo...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
Blend where the condition has been shrunk.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isZeroNode(SDValue Elt)
Returns true if Elt is a constant zero or floating point constant +0.0.
StringRef - Represent a constant reference to a string, i.e.
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement=true)
Returns true of the given offset can be fit into displacement field of the instruction.
APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
TRUNCATE - Completely drop the high bits.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, SDValue Call, SDValue OrigChain)
Replace the original chain operand of the call with load's chain operand and move load below the call...
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
uint64_t getZExtValue() const
This class is used to represent ISD::LOAD nodes.