40 #define DEBUG_TYPE "x86-isel"
42 STATISTIC(NumLoadMoved,
"Number of loads moved below TokenFactor");
52 struct X86ISelAddressMode {
73 unsigned char SymbolFlags;
76 : BaseType(RegBase), Base_FrameIndex(0), Scale(1), IndexReg(), Disp(0),
77 Segment(), GV(nullptr),
CP(nullptr), BlockAddr(nullptr),
ES(nullptr),
80 bool hasSymbolicDisplacement()
const {
81 return GV !=
nullptr ||
CP !=
nullptr ||
ES !=
nullptr ||
82 MCSym !=
nullptr ||
JT != -1 || BlockAddr !=
nullptr;
85 bool hasBaseOrIndexReg()
const {
86 return BaseType == FrameIndexBase ||
87 IndexReg.getNode() !=
nullptr || Base_Reg.getNode() !=
nullptr;
92 bool isRIPRelative()
const {
93 if (BaseType != RegBase)
return false;
95 dyn_cast_or_null<RegisterSDNode>(Base_Reg.getNode()))
96 return RegNode->getReg() == X86::RIP;
105 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
107 dbgs() <<
"X86ISelAddressMode " <<
this <<
'\n';
108 dbgs() <<
"Base_Reg ";
109 if (Base_Reg.getNode())
110 Base_Reg.getNode()->dump();
113 dbgs() <<
" Base.FrameIndex " << Base_FrameIndex <<
'\n'
114 <<
" Scale" << Scale <<
'\n'
116 if (IndexReg.getNode())
117 IndexReg.getNode()->dump();
120 dbgs() <<
" Disp " << Disp <<
'\n'
142 dbgs() <<
" JT" <<
JT <<
" Align" <<
Align <<
'\n';
166 const char *getPassName()
const override {
167 return "X86 DAG->DAG Instruction Selection";
177 void EmitFunctionEntryCode()
override;
181 void PreprocessISelDAG()
override;
183 inline bool immSext8(
SDNode *
N)
const {
184 return isInt<8>(cast<ConstantSDNode>(
N)->getSExtValue());
189 inline bool i64immSExt32(
SDNode *
N)
const {
190 uint64_t v = cast<ConstantSDNode>(
N)->getZExtValue();
191 return (int64_t)v == (int32_t)v;
195 #include "X86GenDAGISel.inc"
202 bool FoldOffsetIntoAddress(uint64_t Offset, X86ISelAddressMode &AM);
203 bool MatchLoadInAddress(
LoadSDNode *N, X86ISelAddressMode &AM);
204 bool MatchWrapper(
SDValue N, X86ISelAddressMode &AM);
205 bool MatchAddress(
SDValue N, X86ISelAddressMode &AM);
206 bool MatchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
208 bool MatchAddressBase(
SDValue N, X86ISelAddressMode &AM);
238 bool SelectInlineAsmMemoryOperand(
const SDValue &Op,
239 unsigned ConstraintID,
240 std::vector<SDValue> &OutOps)
override;
242 void EmitSpecialCodeForMain();
244 inline void getAddressOperands(X86ISelAddressMode &AM,
SDLoc DL,
248 Base = (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
249 ? CurDAG->getTargetFrameIndex(
251 TLI->getPointerTy(CurDAG->getDataLayout()))
253 Scale = getI8Imm(AM.Scale, DL);
258 Disp = CurDAG->getTargetGlobalAddress(AM.GV,
SDLoc(),
262 Disp = CurDAG->getTargetConstantPool(AM.CP,
MVT::i32,
263 AM.Align, AM.Disp, AM.SymbolFlags);
265 assert(!AM.Disp &&
"Non-zero displacement is ignored with ES.");
266 Disp = CurDAG->getTargetExternalSymbol(AM.ES,
MVT::i32, AM.SymbolFlags);
267 }
else if (AM.MCSym) {
268 assert(!AM.Disp &&
"Non-zero displacement is ignored with MCSym.");
269 assert(AM.SymbolFlags == 0 &&
"oo");
270 Disp = CurDAG->getMCSymbol(AM.MCSym,
MVT::i32);
271 }
else if (AM.JT != -1) {
272 assert(!AM.Disp &&
"Non-zero displacement is ignored with JT.");
273 Disp = CurDAG->getTargetJumpTable(AM.JT,
MVT::i32, AM.SymbolFlags);
274 }
else if (AM.BlockAddr)
275 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr,
MVT::i32, AM.Disp,
278 Disp = CurDAG->getTargetConstant(AM.Disp, DL,
MVT::i32);
280 if (AM.Segment.getNode())
281 Segment = AM.Segment;
283 Segment = CurDAG->getRegister(0,
MVT::i32);
289 return CurDAG->getTargetConstant(Imm, DL,
MVT::i8);
295 return CurDAG->getTargetConstant(Imm, DL,
MVT::i32);
302 SDNode *getGlobalBaseReg();
313 return Subtarget->getInstrInfo();
319 bool ComplexPatternFuncMutatesDAG()
const override {
364 if (Imm->getAPIntValue().isSignedIntN(8))
399 "Unexpected chain operand");
451 if (isa<MemSDNode>(Chain.
getNode()) &&
452 cast<MemSDNode>(Chain.
getNode())->writeMem())
463 void X86DAGToDAGISel::PreprocessISelDAG() {
468 E = CurDAG->allnodes_end();
I != E; ) {
477 (Subtarget->is64Bit() ||
478 getTargetMachine().getRelocationModel() !=
Reloc::PIC_)))) {
532 if (SrcIsSSE && DstIsSSE)
535 if (!SrcIsSSE && !DstIsSSE) {
551 MemVT = SrcIsSSE ? SrcVT : DstVT;
553 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
557 SDValue Store = CurDAG->getTruncStore(CurDAG->getEntryNode(), dl,
563 MemVT,
false,
false,
false, 0);
570 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(N, 0), Result);
575 CurDAG->DeleteNode(N);
582 void X86DAGToDAGISel::EmitSpecialCodeForMain() {
583 if (Subtarget->isTargetCygMing()) {
585 auto &DL = CurDAG->getDataLayout();
588 CLI.setChain(CurDAG->getRoot())
590 CurDAG->getExternalSymbol(
"__main", TLI->getPointerTy(DL)),
593 std::pair<SDValue, SDValue> Result = TLI.
LowerCallTo(CLI);
594 CurDAG->setRoot(Result.second);
598 void X86DAGToDAGISel::EmitFunctionEntryCode() {
600 if (
const Function *Fn = MF->getFunction())
601 if (Fn->hasExternalLinkage() && Fn->getName() ==
"main")
602 EmitSpecialCodeForMain();
612 return isInt<31>(Val);
615 bool X86DAGToDAGISel::FoldOffsetIntoAddress(uint64_t Offset,
616 X86ISelAddressMode &AM) {
618 if (Offset != 0 && (AM.ES || AM.MCSym))
620 int64_t Val = AM.Disp + Offset;
622 if (Subtarget->is64Bit()) {
624 AM.hasSymbolicDisplacement()))
628 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
637 bool X86DAGToDAGISel::MatchLoadInAddress(
LoadSDNode *N, X86ISelAddressMode &AM){
647 if (
C->getSExtValue() == 0 && AM.Segment.getNode() ==
nullptr &&
648 Subtarget->isTargetLinux())
651 AM.Segment = CurDAG->getRegister(X86::GS,
MVT::i16);
654 AM.Segment = CurDAG->getRegister(X86::FS,
MVT::i16);
665 bool X86DAGToDAGISel::MatchWrapper(
SDValue N, X86ISelAddressMode &AM) {
668 if (AM.hasSymbolicDisplacement())
682 if (AM.hasBaseOrIndexReg())
685 X86ISelAddressMode Backup = AM;
686 AM.GV =
G->getGlobal();
687 AM.SymbolFlags =
G->getTargetFlags();
688 if (FoldOffsetIntoAddress(
G->getOffset(), AM)) {
693 X86ISelAddressMode Backup = AM;
694 AM.CP =
CP->getConstVal();
695 AM.Align =
CP->getAlignment();
696 AM.SymbolFlags =
CP->getTargetFlags();
697 if (FoldOffsetIntoAddress(
CP->getOffset(), AM)) {
702 AM.ES = S->getSymbol();
703 AM.SymbolFlags = S->getTargetFlags();
704 }
else if (
auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
705 AM.MCSym = S->getMCSymbol();
707 AM.JT = J->getIndex();
708 AM.SymbolFlags = J->getTargetFlags();
710 X86ISelAddressMode Backup = AM;
711 AM.BlockAddr = BA->getBlockAddress();
712 AM.SymbolFlags = BA->getTargetFlags();
713 if (FoldOffsetIntoAddress(BA->getOffset(), AM)) {
721 AM.setBaseReg(CurDAG->getRegister(X86::RIP,
MVT::i64));
728 if (!Subtarget->is64Bit() ||
731 "RIP-relative addressing already handled");
733 AM.GV =
G->getGlobal();
734 AM.Disp +=
G->getOffset();
735 AM.SymbolFlags =
G->getTargetFlags();
737 AM.CP =
CP->getConstVal();
738 AM.Align =
CP->getAlignment();
739 AM.Disp +=
CP->getOffset();
740 AM.SymbolFlags =
CP->getTargetFlags();
742 AM.ES = S->getSymbol();
743 AM.SymbolFlags = S->getTargetFlags();
744 }
else if (
auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
745 AM.MCSym = S->getMCSymbol();
747 AM.JT = J->getIndex();
748 AM.SymbolFlags = J->getTargetFlags();
750 AM.BlockAddr = BA->getBlockAddress();
751 AM.Disp += BA->getOffset();
752 AM.SymbolFlags = BA->getTargetFlags();
764 bool X86DAGToDAGISel::MatchAddress(
SDValue N, X86ISelAddressMode &AM) {
765 if (MatchAddressRecursively(N, AM, 0))
771 AM.BaseType == X86ISelAddressMode::RegBase &&
772 AM.Base_Reg.getNode() ==
nullptr) {
773 AM.Base_Reg = AM.IndexReg;
781 Subtarget->is64Bit() &&
783 AM.BaseType == X86ISelAddressMode::RegBase &&
784 AM.Base_Reg.getNode() ==
nullptr &&
785 AM.IndexReg.getNode() ==
nullptr &&
787 AM.hasSymbolicDisplacement())
788 AM.Base_Reg = CurDAG->getRegister(X86::RIP,
MVT::i64);
813 X86ISelAddressMode &AM) {
820 if (ScaleLog <= 0 || ScaleLog >= 4 ||
821 Mask != (0xffu << ScaleLog))
846 AM.Scale = (1 << ScaleLog);
856 X86ISelAddressMode &AM) {
869 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3)
888 AM.Scale = 1 << ShiftAmt;
889 AM.IndexReg = NewAnd;
923 X86ISelAddressMode &AM) {
934 unsigned AMShiftAmt = MaskTZ;
938 if (AMShiftAmt <= 0 || AMShiftAmt > 3)
return true;
953 bool ReplacingAnyExtend =
false;
960 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits;
961 ReplacingAnyExtend =
true;
963 APInt MaskedHighBits =
965 APInt KnownZero, KnownOne;
967 if (MaskedHighBits != KnownZero)
return true;
972 if (ReplacingAnyExtend) {
996 AM.Scale = 1 << AMShiftAmt;
997 AM.IndexReg = NewSRL;
1001 bool X86DAGToDAGISel::MatchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
1005 dbgs() <<
"MatchAddress: ";
1010 return MatchAddressBase(N, AM);
1015 if (AM.isRIPRelative()) {
1019 if (!(AM.ES || AM.MCSym) && AM.JT != -1)
1023 if (!FoldOffsetIntoAddress(Cst->getSExtValue(), AM))
1031 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
1032 if (
const auto *ESNode = dyn_cast<MCSymbolSDNode>(N.
getOperand(0))) {
1034 AM.MCSym = ESNode->getMCSymbol();
1040 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
1041 if (!FoldOffsetIntoAddress(Val, AM))
1048 if (!MatchWrapper(N, AM))
1053 if (!MatchLoadInAddress(cast<LoadSDNode>(N), AM))
1058 if (AM.BaseType == X86ISelAddressMode::RegBase &&
1059 AM.Base_Reg.getNode() ==
nullptr &&
1061 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
1062 AM.Base_FrameIndex = cast<FrameIndexSDNode>(
N)->getIndex();
1068 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
1073 unsigned Val = CN->getZExtValue();
1078 if (Val == 1 || Val == 2 || Val == 3) {
1079 AM.Scale = 1 << Val;
1085 if (CurDAG->isBaseWithConstantOffset(ShVal)) {
1089 uint64_t Disp = (uint64_t)AddVal->
getSExtValue() << Val;
1090 if (!FoldOffsetIntoAddress(Disp, AM))
1094 AM.IndexReg = ShVal;
1102 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
1135 if (AM.BaseType == X86ISelAddressMode::RegBase &&
1136 AM.Base_Reg.getNode() ==
nullptr &&
1137 AM.IndexReg.getNode() ==
nullptr) {
1140 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
1141 CN->getZExtValue() == 9) {
1142 AM.Scale =
unsigned(CN->getZExtValue())-1;
1150 if (MulVal.getNode()->getOpcode() ==
ISD::ADD && MulVal.hasOneUse() &&
1151 isa<ConstantSDNode>(MulVal.getNode()->getOperand(1))) {
1154 cast<ConstantSDNode>(MulVal.getNode()->getOperand(1));
1155 uint64_t Disp = AddVal->
getSExtValue() * CN->getZExtValue();
1156 if (FoldOffsetIntoAddress(Disp, AM))
1162 AM.IndexReg = AM.Base_Reg =
Reg;
1181 X86ISelAddressMode Backup = AM;
1187 if (AM.IndexReg.getNode() || AM.isRIPRelative()) {
1206 if ((AM.BaseType == X86ISelAddressMode::RegBase &&
1207 AM.Base_Reg.getNode() &&
1208 !AM.Base_Reg.getNode()->hasOneUse()) ||
1209 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
1213 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
1214 ((AM.Disp != 0) && (Backup.Disp == 0)) +
1215 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
1240 X86ISelAddressMode Backup = AM;
1241 if (!MatchAddressRecursively(N.
getOperand(0), AM, Depth+1) &&
1242 !MatchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1))
1247 if (!MatchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1)&&
1248 !MatchAddressRecursively(Handle.getValue().getOperand(0), AM, Depth+1))
1255 if (AM.BaseType == X86ISelAddressMode::RegBase &&
1256 !AM.Base_Reg.getNode() &&
1257 !AM.IndexReg.getNode()) {
1270 if (CurDAG->isBaseWithConstantOffset(N)) {
1271 X86ISelAddressMode Backup = AM;
1275 if (!MatchAddressRecursively(N.
getOperand(0), AM, Depth+1) &&
1287 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
1317 return MatchAddressBase(N, AM);
1322 bool X86DAGToDAGISel::MatchAddressBase(
SDValue N, X86ISelAddressMode &AM) {
1324 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
1326 if (!AM.IndexReg.getNode()) {
1337 AM.BaseType = X86ISelAddressMode::RegBase;
1349 X86ISelAddressMode AM;
1352 if (AddrSpace == 256)
1353 AM.Segment = CurDAG->getRegister(X86::GS,
MVT::i16);
1354 if (AddrSpace == 257)
1355 AM.Segment = CurDAG->getRegister(X86::FS,
MVT::i16);
1361 Scale = getI8Imm(ScalarSize/8, DL);
1364 if (isa<ConstantSDNode>(Base)) {
1365 assert(dyn_cast<ConstantSDNode>(Base)->isNullValue() &&
1366 "Unexpected base in gather/scatter");
1367 Scale = getI8Imm(1, DL);
1368 Base = CurDAG->getRegister(0,
MVT::i32);
1370 if (AM.Segment.getNode())
1371 Segment = AM.Segment;
1373 Segment = CurDAG->getRegister(0,
MVT::i32);
1374 Disp = CurDAG->getTargetConstant(0, DL,
MVT::i32);
1388 X86ISelAddressMode AM;
1398 unsigned AddrSpace =
1399 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
1401 if (AddrSpace == 256)
1402 AM.Segment = CurDAG->getRegister(X86::GS,
MVT::i16);
1403 if (AddrSpace == 257)
1404 AM.Segment = CurDAG->getRegister(X86::FS,
MVT::i16);
1407 if (MatchAddress(N, AM))
1411 if (AM.BaseType == X86ISelAddressMode::RegBase) {
1412 if (!AM.Base_Reg.getNode())
1413 AM.Base_Reg = CurDAG->getRegister(0, VT);
1416 if (!AM.IndexReg.getNode())
1417 AM.IndexReg = CurDAG->getRegister(0, VT);
1419 getAddressOperands(AM,
SDLoc(N), Base, Scale, Index, Disp, Segment);
1430 bool X86DAGToDAGISel::SelectScalarSSELoad(
SDNode *Root,
1434 SDValue &PatternNodeWithChain) {
1441 LoadSDNode *
LD = cast<LoadSDNode>(PatternNodeWithChain);
1442 if (!SelectAddr(LD, LD->
getBasePtr(), Base, Scale, Index, Disp, Segment))
1460 if (!SelectAddr(LD, LD->
getBasePtr(), Base, Scale, Index, Disp, Segment))
1462 PatternNodeWithChain =
SDValue(LD, 0);
1469 bool X86DAGToDAGISel::SelectMOV64Imm32(
SDValue N,
SDValue &Imm) {
1472 if ((uint32_t)ImmVal != (uint64_t)ImmVal)
1475 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(N),
MVT::i64);
1483 "Unexpected node type for MOV32ri64");
1498 bool X86DAGToDAGISel::SelectLEA64_32Addr(
SDValue N,
SDValue &Base,
1501 if (!SelectLEAAddr(N, Base, Scale, Index, Disp, Segment))
1506 if (RN && RN->
getReg() == 0)
1507 Base = CurDAG->getRegister(0,
MVT::i64);
1510 Base =
SDValue(CurDAG->getMachineNode(
1512 CurDAG->getTargetConstant(0, DL,
MVT::i64),
1514 CurDAG->getTargetConstant(X86::sub_32bit, DL,
MVT::i32)),
1519 if (RN && RN->
getReg() == 0)
1520 Index = CurDAG->getRegister(0,
MVT::i64);
1523 "Expect to be extending 32-bit registers for use in LEA");
1524 Index =
SDValue(CurDAG->getMachineNode(
1526 CurDAG->getTargetConstant(0, DL,
MVT::i64),
1528 CurDAG->getTargetConstant(X86::sub_32bit, DL,
1538 bool X86DAGToDAGISel::SelectLEAAddr(
SDValue N,
1542 X86ISelAddressMode AM;
1549 if (MatchAddress(N, AM))
1551 assert (T == AM.Segment);
1555 unsigned Complexity = 0;
1556 if (AM.BaseType == X86ISelAddressMode::RegBase)
1557 if (AM.Base_Reg.getNode())
1560 AM.Base_Reg = CurDAG->getRegister(0, VT);
1561 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
1564 if (AM.IndexReg.getNode())
1567 AM.IndexReg = CurDAG->getRegister(0, VT);
1579 if (AM.hasSymbolicDisplacement()) {
1582 if (Subtarget->is64Bit())
1588 if (AM.Disp && (AM.Base_Reg.getNode() || AM.IndexReg.getNode()))
1592 if (Complexity <= 2)
1595 getAddressOperands(AM,
SDLoc(N), Base, Scale, Index, Disp, Segment);
1600 bool X86DAGToDAGISel::SelectTLSADDRAddr(
SDValue N,
SDValue &Base,
1606 X86ISelAddressMode AM;
1607 AM.GV = GA->getGlobal();
1608 AM.Disp += GA->getOffset();
1609 AM.Base_Reg = CurDAG->getRegister(0, N.
getValueType());
1610 AM.SymbolFlags = GA->getTargetFlags();
1614 AM.IndexReg = CurDAG->getRegister(X86::EBX,
MVT::i32);
1616 AM.IndexReg = CurDAG->getRegister(0,
MVT::i64);
1619 getAddressOperands(AM,
SDLoc(N), Base, Scale, Index, Disp, Segment);
1629 !IsProfitableToFold(N, P, P) ||
1630 !IsLegalToFold(N, P, P, OptLevel))
1633 return SelectAddr(N.
getNode(),
1634 N.
getOperand(1), Base, Scale, Index, Disp, Segment);
1641 SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
1642 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
1643 auto &DL = MF->getDataLayout();
1644 return CurDAG->getRegister(GlobalBaseReg, TLI->
getPointerTy(DL)).getNode();
1686 X86::LOCK_ADD64mi32,
1699 X86::LOCK_SUB64mi32,
1751 X86::LOCK_AND64mi32,
1764 X86::LOCK_XOR64mi32,
1783 if ((int32_t)CNVal != CNVal)
1789 if (CNVal == INT32_MIN)
1794 if (((CNVal == 1) || (CNVal == -1)) && !Subtarget->
slowIncDec()) {
1795 Op = (CNVal == 1) ?
INC :
DEC;
1831 SDNode *X86DAGToDAGISel::SelectAtomicLoadArith(
SDNode *Node,
MVT NVT) {
1843 SDValue Base, Scale, Index, Disp, Segment;
1844 if (!SelectAddr(Node, Ptr, Base, Scale, Index, Disp, Segment))
1872 default:
return nullptr;
1901 else if (i64immSExt32(Val.
getNode()))
1910 assert(Opc != 0 &&
"Invalid arith lock transform!");
1915 SDValue Ops[] = { Base, Scale, Index, Disp, Segment, Chain };
1918 SDValue Ops[] = { Base, Scale, Index, Disp, Segment, Val, Chain };
1924 MemOp[0] = cast<MemSDNode>(Node)->getMemOperand();
1925 cast<MachineSDNode>(
Ret)->setMemRefs(MemOp, MemOp + 1);
1933 return CurDAG->getMergeValues(RetVals, dl).getNode();
1941 UE = N->
use_end(); UI != UE; ++UI) {
1946 if (cast<RegisterSDNode>(UI->getOperand(1))->
getReg() !=
1951 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
1953 if (FlagUI.getUse().getResNo() != 1)
continue;
1955 if (!FlagUI->isMachineOpcode())
return false;
1957 switch (FlagUI->getMachineOpcode()) {
1959 case X86::SETAr:
case X86::SETAEr:
case X86::SETBr:
case X86::SETBEr:
1960 case X86::SETEr:
case X86::SETNEr:
case X86::SETPr:
case X86::SETNPr:
1961 case X86::SETAm:
case X86::SETAEm:
case X86::SETBm:
case X86::SETBEm:
1962 case X86::SETEm:
case X86::SETNEm:
case X86::SETPm:
case X86::SETNPm:
1963 case X86::JA_1:
case X86::JAE_1:
case X86::JB_1:
case X86::JBE_1:
1964 case X86::JE_1:
case X86::JNE_1:
case X86::JP_1:
case X86::JNP_1:
1965 case X86::CMOVA16rr:
case X86::CMOVA16rm:
1966 case X86::CMOVA32rr:
case X86::CMOVA32rm:
1967 case X86::CMOVA64rr:
case X86::CMOVA64rm:
1968 case X86::CMOVAE16rr:
case X86::CMOVAE16rm:
1969 case X86::CMOVAE32rr:
case X86::CMOVAE32rm:
1970 case X86::CMOVAE64rr:
case X86::CMOVAE64rm:
1971 case X86::CMOVB16rr:
case X86::CMOVB16rm:
1972 case X86::CMOVB32rr:
case X86::CMOVB32rm:
1973 case X86::CMOVB64rr:
case X86::CMOVB64rm:
1974 case X86::CMOVBE16rr:
case X86::CMOVBE16rm:
1975 case X86::CMOVBE32rr:
case X86::CMOVBE32rm:
1976 case X86::CMOVBE64rr:
case X86::CMOVBE64rm:
1977 case X86::CMOVE16rr:
case X86::CMOVE16rm:
1978 case X86::CMOVE32rr:
case X86::CMOVE32rm:
1979 case X86::CMOVE64rr:
case X86::CMOVE64rm:
1980 case X86::CMOVNE16rr:
case X86::CMOVNE16rm:
1981 case X86::CMOVNE32rr:
case X86::CMOVNE32rm:
1982 case X86::CMOVNE64rr:
case X86::CMOVNE64rm:
1983 case X86::CMOVNP16rr:
case X86::CMOVNP16rm:
1984 case X86::CMOVNP32rr:
case X86::CMOVNP32rm:
1985 case X86::CMOVNP64rr:
case X86::CMOVNP64rm:
1986 case X86::CMOVP16rr:
case X86::CMOVP16rm:
1987 case X86::CMOVP32rr:
case X86::CMOVP32rm:
1988 case X86::CMOVP64rr:
case X86::CMOVP64rm:
1991 default:
return false;
2009 if (StoredVal.
getResNo() != 0)
return false;
2023 LoadNode = cast<LoadSDNode>(
Load);
2043 bool ChainCheck =
false;
2062 UE = UI->
use_end(); UI != UE; ++UI) {
2063 if (UI.getUse().getResNo() != 0)
2065 if (UI->getNodeId() > LoadId)
2088 if (LdVT ==
MVT::i64)
return X86::DEC64m;
2089 if (LdVT ==
MVT::i32)
return X86::DEC32m;
2090 if (LdVT ==
MVT::i16)
return X86::DEC16m;
2091 if (LdVT ==
MVT::i8)
return X86::DEC8m;
2093 assert(Opc ==
X86ISD::INC &&
"unrecognized opcode");
2094 if (LdVT ==
MVT::i64)
return X86::INC64m;
2095 if (LdVT ==
MVT::i32)
return X86::INC32m;
2096 if (LdVT ==
MVT::i16)
return X86::INC16m;
2097 if (LdVT ==
MVT::i8)
return X86::INC8m;
2104 SDNode *X86DAGToDAGISel::SelectGather(
SDNode *Node,
unsigned Opc) {
2124 Disp, Segment, VMask, Chain};
2125 SDNode *ResNode = CurDAG->getMachineNode(Opc, DL, VTs, Ops);
2152 unsigned IntNo = cast<ConstantSDNode>(Node->
getOperand(1))->getZExtValue();
2155 case Intrinsic::x86_avx2_gather_d_pd:
2156 case Intrinsic::x86_avx2_gather_d_pd_256:
2157 case Intrinsic::x86_avx2_gather_q_pd:
2158 case Intrinsic::x86_avx2_gather_q_pd_256:
2159 case Intrinsic::x86_avx2_gather_d_ps:
2160 case Intrinsic::x86_avx2_gather_d_ps_256:
2161 case Intrinsic::x86_avx2_gather_q_ps:
2162 case Intrinsic::x86_avx2_gather_q_ps_256:
2163 case Intrinsic::x86_avx2_gather_d_q:
2164 case Intrinsic::x86_avx2_gather_d_q_256:
2165 case Intrinsic::x86_avx2_gather_q_q:
2166 case Intrinsic::x86_avx2_gather_q_q_256:
2167 case Intrinsic::x86_avx2_gather_d_d:
2168 case Intrinsic::x86_avx2_gather_d_d_256:
2169 case Intrinsic::x86_avx2_gather_q_d:
2170 case Intrinsic::x86_avx2_gather_q_d_256: {
2171 if (!Subtarget->hasAVX2())
2176 case Intrinsic::x86_avx2_gather_d_pd: Opc = X86::VGATHERDPDrm;
break;
2177 case Intrinsic::x86_avx2_gather_d_pd_256: Opc = X86::VGATHERDPDYrm;
break;
2178 case Intrinsic::x86_avx2_gather_q_pd: Opc = X86::VGATHERQPDrm;
break;
2179 case Intrinsic::x86_avx2_gather_q_pd_256: Opc = X86::VGATHERQPDYrm;
break;
2180 case Intrinsic::x86_avx2_gather_d_ps: Opc = X86::VGATHERDPSrm;
break;
2181 case Intrinsic::x86_avx2_gather_d_ps_256: Opc = X86::VGATHERDPSYrm;
break;
2182 case Intrinsic::x86_avx2_gather_q_ps: Opc = X86::VGATHERQPSrm;
break;
2183 case Intrinsic::x86_avx2_gather_q_ps_256: Opc = X86::VGATHERQPSYrm;
break;
2184 case Intrinsic::x86_avx2_gather_d_q: Opc = X86::VPGATHERDQrm;
break;
2185 case Intrinsic::x86_avx2_gather_d_q_256: Opc = X86::VPGATHERDQYrm;
break;
2186 case Intrinsic::x86_avx2_gather_q_q: Opc = X86::VPGATHERQQrm;
break;
2187 case Intrinsic::x86_avx2_gather_q_q_256: Opc = X86::VPGATHERQQYrm;
break;
2188 case Intrinsic::x86_avx2_gather_d_d: Opc = X86::VPGATHERDDrm;
break;
2189 case Intrinsic::x86_avx2_gather_d_d_256: Opc = X86::VPGATHERDDYrm;
break;
2190 case Intrinsic::x86_avx2_gather_q_d: Opc = X86::VPGATHERQDrm;
break;
2191 case Intrinsic::x86_avx2_gather_q_d_256: Opc = X86::VPGATHERQDYrm;
break;
2193 SDNode *RetVal = SelectGather(Node, Opc);
2203 return getGlobalBaseReg();
2207 SDValue VSelect = CurDAG->getNode(
2210 ReplaceUses(
SDValue(Node, 0), VSelect);
2211 SelectCode(VSelect.
getNode());
2220 SDNode *RetVal = SelectAtomicLoadArith(Node, NVT);
2242 if (!Cst || !ShlCst)
2246 uint64_t ShlVal = ShlCst->getZExtValue();
2250 uint64_t RemovedBitsMask = (1ULL << ShlVal) - 1;
2251 if (Opcode !=
ISD::AND && (Val & RemovedBitsMask) != 0)
2254 unsigned ShlOp, AddOp, Op;
2274 ShlOp = X86::SHL32ri;
2275 AddOp = X86::ADD32rr;
2279 case ISD::AND: Op = X86::AND32ri8;
break;
2280 case ISD::OR: Op = X86::OR32ri8;
break;
2281 case ISD::XOR: Op = X86::XOR32ri8;
break;
2286 ShlOp = X86::SHL64ri;
2287 AddOp = X86::ADD64rr;
2291 case ISD::AND: Op = CstVT==
MVT::i8? X86::AND64ri8 : X86::AND64ri32;
break;
2292 case ISD::OR: Op = CstVT==
MVT::i8? X86::OR64ri8 : X86::OR64ri32;
break;
2293 case ISD::XOR: Op = CstVT==
MVT::i8? X86::XOR64ri8 : X86::XOR64ri32;
break;
2299 SDValue NewCst = CurDAG->getTargetConstant(Val >> ShlVal, dl, CstVT);
2300 SDNode *New = CurDAG->getMachineNode(Op, dl, NVT, N0->
getOperand(0),NewCst);
2302 return CurDAG->SelectNodeTo(Node, AddOp, NVT,
SDValue(New, 0),
2304 return CurDAG->SelectNodeTo(Node, ShlOp, NVT,
SDValue(New, 0),
2305 getI8Imm(ShlVal, dl));
2314 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::AL,
2319 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
2333 case MVT::i8: LoReg = X86::AL; Opc = X86::MUL8r;
break;
2334 case MVT::i16: LoReg = X86::AX; Opc = X86::MUL16r;
break;
2335 case MVT::i32: LoReg = X86::EAX; Opc = X86::MUL32r;
break;
2336 case MVT::i64: LoReg = X86::RAX; Opc = X86::MUL64r;
break;
2339 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
2344 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
2358 bool hasBMI2 = Subtarget->hasBMI2();
2362 case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m;
break;
2363 case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m;
break;
2364 case MVT::i32: Opc = hasBMI2 ? X86::MULX32rr : X86::MUL32r;
2365 MOpc = hasBMI2 ? X86::MULX32rm : X86::MUL32m;
break;
2366 case MVT::i64: Opc = hasBMI2 ? X86::MULX64rr : X86::MUL64r;
2367 MOpc = hasBMI2 ? X86::MULX64rm : X86::MUL64m;
break;
2372 case MVT::i8: Opc = X86::IMUL8r; MOpc = X86::IMUL8m;
break;
2373 case MVT::i16: Opc = X86::IMUL16r; MOpc = X86::IMUL16m;
break;
2374 case MVT::i32: Opc = X86::IMUL32r; MOpc = X86::IMUL32m;
break;
2375 case MVT::i64: Opc = X86::IMUL64r; MOpc = X86::IMUL64m;
break;
2379 unsigned SrcReg, LoReg, HiReg;
2384 SrcReg = LoReg = X86::AL; HiReg = X86::AH;
2388 SrcReg = LoReg = X86::AX; HiReg = X86::DX;
2392 SrcReg = LoReg = X86::EAX; HiReg = X86::EDX;
2396 SrcReg = LoReg = X86::RAX; HiReg = X86::RDX;
2399 SrcReg = X86::EDX; LoReg = HiReg = 0;
2402 SrcReg = X86::RDX; LoReg = HiReg = 0;
2406 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
2407 bool foldedLoad = TryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
2410 foldedLoad = TryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
2415 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, SrcReg,
2423 if (MOpc == X86::MULX32rm || MOpc == X86::MULX64rm) {
2425 SDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
2432 SDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
2438 ReplaceUses(N1.
getValue(1), Chain);
2440 SDValue Ops[] = { N1, InFlag };
2441 if (Opc == X86::MULX32rr || Opc == X86::MULX64rr) {
2443 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
2449 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
2455 if (HiReg == X86::AH && Subtarget->is64Bit() &&
2457 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
2462 if (!
SDValue(Node, 0).use_empty())
2464 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl,
MVT::i8, Result));
2467 Result =
SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16,
2469 CurDAG->getTargetConstant(8, dl,
MVT::i8)),
2473 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl,
MVT::i8, Result));
2478 assert(LoReg &&
"Register for low half is not defined!");
2479 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg, NVT,
2483 ReplaceUses(
SDValue(Node, 0), ResLo);
2489 assert(HiReg &&
"Register for high half is not defined!");
2490 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg, NVT,
2494 ReplaceUses(
SDValue(Node, 1), ResHi);
2511 switch (NVT.SimpleTy) {
2513 case MVT::i8: Opc = X86::DIV8r; MOpc = X86::DIV8m;
break;
2514 case MVT::i16: Opc = X86::DIV16r; MOpc = X86::DIV16m;
break;
2515 case MVT::i32: Opc = X86::DIV32r; MOpc = X86::DIV32m;
break;
2516 case MVT::i64: Opc = X86::DIV64r; MOpc = X86::DIV64m;
break;
2519 switch (NVT.SimpleTy) {
2521 case MVT::i8: Opc = X86::IDIV8r; MOpc = X86::IDIV8m;
break;
2522 case MVT::i16: Opc = X86::IDIV16r; MOpc = X86::IDIV16m;
break;
2523 case MVT::i32: Opc = X86::IDIV32r; MOpc = X86::IDIV32m;
break;
2524 case MVT::i64: Opc = X86::IDIV64r; MOpc = X86::IDIV64m;
break;
2528 unsigned LoReg, HiReg, ClrReg;
2529 unsigned SExtOpcode;
2530 switch (NVT.SimpleTy) {
2533 LoReg = X86::AL; ClrReg = HiReg = X86::AH;
2534 SExtOpcode = X86::CBW;
2537 LoReg = X86::AX; HiReg = X86::DX;
2539 SExtOpcode = X86::CWD;
2542 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX;
2543 SExtOpcode = X86::CDQ;
2546 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX;
2547 SExtOpcode = X86::CQO;
2551 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
2552 bool foldedLoad = TryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
2553 bool signBitIsZero = CurDAG->SignBitIsZero(N0);
2556 if (NVT ==
MVT::i8 && (!isSigned || signBitIsZero)) {
2559 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Move, Chain;
2560 if (TryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
2566 ReplaceUses(N0.
getValue(1), Chain);
2570 Chain = CurDAG->getEntryNode();
2572 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EAX, Move,
SDValue());
2576 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
2577 LoReg, N0,
SDValue()).getValue(1);
2578 if (isSigned && !signBitIsZero) {
2584 SDValue ClrNode =
SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, NVT), 0);
2585 switch (NVT.SimpleTy) {
2588 SDValue(CurDAG->getMachineNode(
2590 CurDAG->getTargetConstant(X86::sub_16bit, dl,
2598 SDValue(CurDAG->getMachineNode(
2600 CurDAG->getTargetConstant(0, dl,
MVT::i64), ClrNode,
2601 CurDAG->getTargetConstant(X86::sub_32bit, dl,
2609 InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg,
2610 ClrNode, InFlag).getValue(1);
2636 unsigned AHExtOpcode =
2637 isSigned ? X86::MOVSX32_NOREXrr8 : X86::MOVZX32_NOREXrr8;
2639 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl,
MVT::i32,
2650 "Unexpected i64 sext of h-register");
2652 SDValue(CurDAG->getMachineNode(
2654 CurDAG->getTargetConstant(0, dl,
MVT::i64), Result,
2655 CurDAG->getTargetConstant(X86::sub_32bit, dl,
2661 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl,
MVT::i8, Result);
2663 ReplaceUses(
SDValue(Node, 1), Result);
2664 DEBUG(
dbgs() <<
"=> "; Result.getNode()->dump(CurDAG);
dbgs() <<
'\n');
2668 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
2669 LoReg, NVT, InFlag);
2671 ReplaceUses(
SDValue(Node, 0), Result);
2676 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
2677 HiReg, NVT, InFlag);
2679 ReplaceUses(
SDValue(Node, 1), Result);
2717 if (!Subtarget->is64Bit()) {
2720 case MVT::i32: TRC = &X86::GR32_ABCDRegClass;
break;
2721 case MVT::i16: TRC = &X86::GR16_ABCDRegClass;
break;
2730 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl,
2734 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri, dl,
MVT::i32,
2755 case MVT::i64: TRC = &X86::GR64_ABCDRegClass;
break;
2756 case MVT::i32: TRC = &X86::GR32_ABCDRegClass;
break;
2757 case MVT::i16: TRC = &X86::GR16_ABCDRegClass;
break;
2765 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl,
2771 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri_NOREX, dl,
2790 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl,
2794 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST16ri, dl,
MVT::i32,
2804 if ((C->
getZExtValue() & ~UINT64_C(0xffffffff)) == 0 &&
2813 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_32bit, dl,
2817 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST32ri, dl,
MVT::i32,
2848 unsigned Opc = StoredVal->getOpcode();
2853 LoadNode, InputChain))
2856 SDValue Base, Scale, Index, Disp, Segment;
2857 if (!SelectAddr(LoadNode, LoadNode->
getBasePtr(),
2858 Base, Scale, Index, Disp, Segment))
2864 const SDValue Ops[] = { Base, Scale, Index, Disp, Segment, InputChain };
2873 ReplaceUses(
SDValue(StoredVal.getNode(), 1),
SDValue(Result, 0));
2879 SDNode *ResNode = SelectCode(Node);
2882 if (ResNode ==
nullptr || ResNode == Node)
2885 ResNode->
dump(CurDAG);
2891 bool X86DAGToDAGISel::
2892 SelectInlineAsmMemoryOperand(
const SDValue &Op,
unsigned ConstraintID,
2893 std::vector<SDValue> &OutOps) {
2894 SDValue Op0, Op1, Op2, Op3, Op4;
2895 switch (ConstraintID) {
2906 if (!SelectAddr(
nullptr, Op, Op0, Op1, Op2, Op3, Op4))
2911 OutOps.push_back(Op0);
2912 OutOps.push_back(Op1);
2913 OutOps.push_back(Op2);
2914 OutOps.push_back(Op3);
2915 OutOps.push_back(Op4);
2924 return new X86DAGToDAGISel(TM, OptLevel);
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
bool isInt< 32 >(int64_t x)
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq)
isCalleeLoad - Return true if call address is a load and it can be moved below CALLSEQ_START and the ...
void push_back(const T &Elt)
SDValue getValue(unsigned R) const
const SDValue & getValue() const
void dump() const
Dump this node, for debugging.
STATISTIC(NumFunctions,"Total number of functions")
void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd)
Assign this MachineSDNodes's memory reference descriptor list.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
static bool FoldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
Carry-setting nodes for multiple precision addition and subtraction.
AtomicOpc
Atomic opcode table.
void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in the KnownZero/KnownO...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getID() const
getID() - Return the register class ID number.
unsigned getSizeInBits() const
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
unsigned getNumOperands() const
const SDValue & getOperand(unsigned Num) const
void setNodeId(int Id)
Set unique node id.
const SDValue & getBasePtr() const
std::size_t countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1...
unsigned getResNo() const
get the index which selects a specific result in the SDNode
static SDValue getAtomicLoadArithTargetConstant(SelectionDAG *CurDAG, SDLoc dl, enum AtomicOpc &Op, MVT NVT, SDValue Val, const X86Subtarget *Subtarget)
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
GlobalBaseReg - On Darwin, this node represents the result of the mflr at function entry...
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
X86 compare and logical compare instructions.
BlockAddress - The address of a basic block.
MachineMemOperand - A description of a memory reference used in the backend.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Shift and rotation operations.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
std::size_t countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isInt< 8 >(int64_t x)
CopyToReg - This node has three operands: a chain, a register number to set to this value...
Reg
All possible values of the reg field in the ModR/M byte.
static bool HasNoSignedComparisonUses(SDNode *N)
HasNoSignedComparisonUses - Test whether the given X86ISD::CMP node has any uses which require the SF...
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence, and carry arbitrary information that target might want to know.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
static bool FoldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
static bool isLoadIncOrDecStore(StoreSDNode *StoreNode, unsigned Opc, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
isLoadIncOrDecStore - Check whether or not the chain ending in StoreNode is suitable for doing the {l...
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Simple integer binary arithmetic operators.
const SDValue & getBasePtr() const
static const uint16_t AtomicOpcTbl[AtomicOpcEnd][AtomicSzEnd]
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification, or lowering of the constant.
EVT getMemoryVT() const
Return the type of the in-memory value.
This class is used to represent ISD::STORE nodes.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
SDNode * getNode() const
get the SDNode which holds the desired result
IMPLICIT_DEF - This is the MachineInstr-level equivalent of undef.
unsigned getScalarSizeInBits() const
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
MVT - Machine Value Type.
const SDValue & getOperand(unsigned i) const
bool isNonTemporal() const
This is an important base class in LLVM.
bool isVector() const
isVector - Return true if this is a vector value type.
Carry-using nodes for multiple precision addition and subtraction.
This is a base class used to represent MGATHER and MSCATTER nodes.
static Type * getVoidTy(LLVMContext &C)
This class provides iterator support for SDUse operands that use a specific SDNode.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
SDValue getTargetConstant(uint64_t Val, SDLoc DL, EVT VT, bool isOpaque=false)
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
unsigned getOpcode() const
FunctionPass class - This class is used to implement most global optimizations.
const SDValue & getBasePtr() const
On Darwin, this node represents the result of the popl at function entry, used for PIC code...
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
EVT - Extended Value Type.
std::vector< ArgListEntry > ArgListTy
This structure contains all information that is necessary for lowering calls.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
MachinePointerInfo - This class contains a discriminated union of information about pointers in memor...
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getOffset() const
These operations represent an abstract X86 call instruction, which includes a bunch of information...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
EXTRACT_SUBREG - This instruction takes two operands: a register that has subregisters, and a subregister index.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
An SDNode that represents everything that will be needed to construct a MachineInstr.
const SDValue & getChain() const
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
unsigned getAddrSpace() const
getAddrSpace - Return the LLVM IR address space number that this pointer points into.
Represents one node in the SelectionDAG.
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
static void InsertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N)
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Class for arbitrary precision integers.
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
int64_t getSExtValue() const
op_iterator op_begin() const
static use_iterator use_end()
ZERO_EXTEND - Used for integer types, zeroing the new bits.
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
ANY_EXTEND - Used for integer types. The high bits are undefined.
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
static bool isDispSafeForFrameIndex(int64_t Val)
uint64_t getConstantOperandVal(unsigned i) const
static unsigned getFusedLdStOpcode(EVT &LdVT, unsigned Opc)
getFusedLdStOpcode - Get the appropriate X86 opcode for an in memory increment or decrement...
Bitwise operators - logical and, logical or, logical xor.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
A wrapper node for TargetConstantPool, TargetExternalSymbol, and TargetGlobalAddress.
FunctionPass * createX86ISelDag(X86TargetMachine &TM, CodeGenOpt::Level OptLevel)
createX86ISelDag - This pass converts a legalized DAG into a X86-specific DAG, ready for instruction ...
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
COPY_TO_REGCLASS - This instruction is a placeholder for a plain register-to-register copy into a spe...
bool isOperandOf(const SDNode *N) const
isOperand - Return true if this node is an operand of N.
static bool FoldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static void MoveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, SDValue Call, SDValue OrigChain)
MoveBelowCallOrigChain - Replace the original chain operand of the call with load's chain operand and...
op_iterator op_end() const
const SDValue & getOffset() const
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
const SDValue & getIndex() const
int getNodeId() const
Return the unique node id.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isTarget=false, bool isOpaque=false)
This class is used to form a handle around another node that is persistent and is updated across invo...
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
SDValue getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
Blend where the condition has been shrunk.
C - The default llvm calling convention, compatible with C.
bool isZeroNode(SDValue Elt)
Returns true if Elt is a constant zero or floating point constant +0.0.
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement=true)
Returns true of the given offset can be fit into displacement field of the instruction.
SUBREG_TO_REG - This instruction is similar to INSERT_SUBREG except that the first operand is an imme...
TRUNCATE - Completely drop the high bits.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
uint64_t getZExtValue() const
This class is used to represent ISD::LOAD nodes.