23#include "llvm/Config/llvm-config.h"
28#include "llvm/IR/IntrinsicsX86.h"
39#define DEBUG_TYPE "x86-isel"
40#define PASS_NAME "X86 DAG->DAG Instruction Selection"
42STATISTIC(NumLoadMoved,
"Number of loads moved below TokenFactor");
45 cl::desc(
"Enable setting constant bits to reduce size of mask immediates"),
49 "x86-promote-anyext-load",
cl::init(
true),
61 struct X86ISelAddressMode {
69 int Base_FrameIndex = 0;
78 const char *ES =
nullptr;
83 bool NegateIndex =
false;
85 X86ISelAddressMode() =
default;
87 bool hasSymbolicDisplacement()
const {
88 return GV !=
nullptr ||
CP !=
nullptr || ES !=
nullptr ||
89 MCSym !=
nullptr ||
JT != -1 || BlockAddr !=
nullptr;
92 bool hasBaseOrIndexReg()
const {
99 if (
BaseType != RegBase)
return false;
101 dyn_cast_or_null<RegisterSDNode>(Base_Reg.
getNode()))
102 return RegNode->getReg() == X86::RIP;
111#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
113 dbgs() <<
"X86ISelAddressMode " <<
this <<
'\n';
114 dbgs() <<
"Base_Reg ";
120 dbgs() <<
" Base.FrameIndex " << Base_FrameIndex <<
'\n';
121 dbgs() <<
" Scale " << Scale <<
'\n'
129 dbgs() <<
" Disp " << Disp <<
'\n'
151 dbgs() <<
" JT" <<
JT <<
" Align" << Alignment.
value() <<
'\n';
171 bool IndirectTlsSegRefs;
174 X86DAGToDAGISel() =
delete;
178 OptForMinSize(
false), IndirectTlsSegRefs(
false) {}
184 "indirect-tls-seg-refs");
189 "OptForMinSize implies OptForSize");
201#include "X86GenDAGISel.inc"
206 bool foldOffsetIntoAddress(
uint64_t Offset, X86ISelAddressMode &AM);
207 bool matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
208 bool AllowSegmentRegForX32 =
false);
209 bool matchWrapper(
SDValue N, X86ISelAddressMode &AM);
210 bool matchAddress(
SDValue N, X86ISelAddressMode &AM);
211 bool matchVectorAddress(
SDValue N, X86ISelAddressMode &AM);
212 bool matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
unsigned Depth);
215 bool matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
217 bool matchVectorAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
219 bool matchAddressBase(
SDValue N, X86ISelAddressMode &AM);
248 return tryFoldLoad(
P,
P,
N,
Base, Scale,
Index, Disp, Segment);
256 bool isProfitableToFormMaskedOp(
SDNode *
N)
const;
261 std::vector<SDValue> &OutOps)
override;
263 void emitSpecialCodeForMain();
265 inline void getAddressOperands(X86ISelAddressMode &AM,
const SDLoc &
DL,
269 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
270 Base = CurDAG->getTargetFrameIndex(
271 AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout()));
272 else if (AM.Base_Reg.getNode())
275 Base = CurDAG->getRegister(0, VT);
277 Scale = getI8Imm(AM.Scale,
DL);
279#define GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC)
281 if (AM.NegateIndex) {
289 if (AM.IndexReg.getNode())
292 Index = CurDAG->getRegister(0, VT);
297 Disp = CurDAG->getTargetGlobalAddress(AM.GV,
SDLoc(),
301 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment,
302 AM.Disp, AM.SymbolFlags);
304 assert(!AM.Disp &&
"Non-zero displacement is ignored with ES.");
305 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
306 }
else if (AM.MCSym) {
307 assert(!AM.Disp &&
"Non-zero displacement is ignored with MCSym.");
308 assert(AM.SymbolFlags == 0 &&
"oo");
309 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32);
310 }
else if (AM.JT != -1) {
311 assert(!AM.Disp &&
"Non-zero displacement is ignored with JT.");
312 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
313 }
else if (AM.BlockAddr)
314 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
317 Disp = CurDAG->getTargetConstant(AM.Disp,
DL, MVT::i32);
319 if (AM.Segment.getNode())
320 Segment = AM.Segment;
322 Segment = CurDAG->getRegister(0, MVT::i16);
331 bool shouldAvoidImmediateInstFormsForSize(
SDNode *
N)
const {
337 if (!CurDAG->shouldOptForSize())
347 if (
User->isMachineOpcode()) {
370 auto *
C = dyn_cast<ConstantSDNode>(
N);
371 if (
C && isInt<8>(
C->getSExtValue()))
391 (RegNode = dyn_cast_or_null<RegisterSDNode>(
393 if ((RegNode->
getReg() == X86::ESP) ||
394 (RegNode->
getReg() == X86::RSP))
403 return (UseCount > 1);
408 return CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
413 return CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
418 return CurDAG->getTargetConstant(Imm,
DL, MVT::i64);
423 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
425 MVT VecVT =
N->getOperand(0).getSimpleValueType();
431 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
433 MVT VecVT =
N->getSimpleValueType(0);
437 SDValue getPermuteVINSERTCommutedImmediate(
SDNode *
N,
unsigned VecWidth,
439 assert(VecWidth == 128 &&
"Unexpected vector width");
441 MVT VecVT =
N->getSimpleValueType(0);
443 assert((InsertIdx == 0 || InsertIdx == 1) &&
"Bad insertf128 index");
446 return getI8Imm(InsertIdx ? 0x02 : 0x30,
DL);
451 MVT VT =
N->getSimpleValueType(0);
454 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
456 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
457 if (VT == MVT::i64) {
459 CurDAG->getMachineNode(
460 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
461 CurDAG->getTargetConstant(0, dl, MVT::i64), Zero,
462 CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)),
467 unsigned Opcode =
N->getOpcode();
469 "Unexpected opcode for SBB materialization");
470 unsigned FlagOpIndex = Opcode ==
X86ISD::SBB ? 2 : 1;
472 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
473 N->getOperand(FlagOpIndex),
SDValue());
477 unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr;
478 MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
479 VTs = CurDAG->getVTList(SBBVT, MVT::i32);
481 CurDAG->getMachineNode(Opc, dl, VTs,
482 {Zero, Zero, EFLAGS, EFLAGS.getValue(1)}),
488 bool isUnneededShiftMask(
SDNode *
N,
unsigned Width)
const {
490 const APInt &Val =
N->getConstantOperandAPInt(1);
495 APInt Mask = Val | CurDAG->computeKnownBits(
N->getOperand(0)).Zero;
496 return Mask.countr_one() >= Width;
502 SDNode *getGlobalBaseReg();
513 return Subtarget->getInstrInfo();
526 bool isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const;
530 if (!
N->isNonTemporal())
533 unsigned StoreSize =
N->getMemoryVT().getStoreSize();
535 if (
N->getAlign().value() < StoreSize)
544 return Subtarget->hasSSE41();
546 return Subtarget->hasAVX2();
548 return Subtarget->hasAVX512();
552 bool foldLoadStoreIntoMemOperand(
SDNode *
Node);
555 bool shrinkAndImmediate(
SDNode *
N);
556 bool isMaskZeroExtended(
SDNode *
N)
const;
557 bool tryShiftAmountMod(
SDNode *
N);
558 bool tryShrinkShlLogicImm(
SDNode *
N);
564 bool tryMatchBitSelect(
SDNode *
N);
566 MachineSDNode *emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
568 MachineSDNode *emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
572 bool tryOptimizeRem8Extend(
SDNode *
N);
574 bool onlyUsesZeroFlag(
SDValue Flags)
const;
575 bool hasNoSignFlagUses(
SDValue Flags)
const;
576 bool hasNoCarryFlagUses(
SDValue Flags)
const;
585 ID,
std::make_unique<X86DAGToDAGISel>(tm, OptLevel)) {}
589char X86DAGToDAGISelLegacy::ID = 0;
596 unsigned Opcode =
N->getOpcode();
603 EVT OpVT =
N->getOperand(0).getValueType();
607 OpVT =
N->getOperand(1).getValueType();
609 return Subtarget->hasVLX();
623bool X86DAGToDAGISel::isMaskZeroExtended(
SDNode *
N)
const {
636 if (OptLevel == CodeGenOptLevel::None)
646 if (useNonTemporalLoad(cast<LoadSDNode>(
N)))
651 switch (
U->getOpcode()) {
677 if (
auto *Imm = dyn_cast<ConstantSDNode>(Op1)) {
678 if (
Imm->getAPIntValue().isSignedIntN(8))
687 Imm->getAPIntValue().getBitWidth() == 64 &&
688 Imm->getAPIntValue().isIntN(32))
695 (
Imm->getAPIntValue() == UINT8_MAX ||
696 Imm->getAPIntValue() == UINT16_MAX ||
697 Imm->getAPIntValue() == UINT32_MAX))
703 (-
Imm->getAPIntValue()).isSignedIntN(8))
707 (-
Imm->getAPIntValue()).isSignedIntN(8) &&
708 hasNoCarryFlagUses(
SDValue(U, 1)))
733 if (
U->getOperand(0).getOpcode() ==
ISD::SHL &&
737 if (
U->getOperand(1).getOpcode() ==
ISD::SHL &&
745 auto *
C = dyn_cast<ConstantSDNode>(U0.
getOperand(0));
746 if (
C &&
C->getSExtValue() == -2)
751 auto *
C = dyn_cast<ConstantSDNode>(U1.
getOperand(0));
752 if (
C &&
C->getSExtValue() == -2)
766 if (isa<ConstantSDNode>(
U->getOperand(1)))
787bool X86DAGToDAGISel::isProfitableToFormMaskedOp(
SDNode *
N)
const {
790 "Unexpected opcode!");
795 return N->getOperand(1).hasOneUse();
804 if (Chain.
getNode() == Load.getNode())
808 "Unexpected chain operand");
822 Load.getOperand(1), Load.getOperand(2));
826 Ops.
append(Call->op_begin() + 1, Call->op_end());
840 if (Callee.getNode() == Chain.
getNode() || !Callee.hasOneUse())
842 auto *LD = dyn_cast<LoadSDNode>(Callee.getNode());
860 if (isa<MemSDNode>(Chain.
getNode()) &&
861 cast<MemSDNode>(Chain.
getNode())->writeMem())
867 Callee.getValue(1).hasOneUse())
875 if ((Imm & 0x00FFFFFF) != 0x0F1EFA)
878 uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64,
879 0x65, 0x66, 0x67, 0xf0, 0xf2};
882 uint8_t Byte = (Imm >> i) & 0xFF;
894 return (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v64i8);
897void X86DAGToDAGISel::PreprocessISelDAG() {
898 bool MadeChange =
false;
900 E = CurDAG->allnodes_end();
I != E; ) {
919 MVT VT =
N->getSimpleValueType(0);
920 int64_t
Imm = cast<ConstantSDNode>(
N)->getSExtValue();
921 int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB;
928 SDValue Complement = CurDAG->getConstant(~Imm, dl, VT,
false,
true);
929 Complement = CurDAG->getNOT(dl, Complement, VT);
931 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Complement);
941 if (
N->getOpcode() ==
X86ISD::AND && !
N->hasAnyUseOfValue(1)) {
943 N->getOperand(0),
N->getOperand(1));
945 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
969 auto mayPreventLoadFold = [&]() {
971 N->getOpcode() ==
ISD::ADD && Subtarget->hasAVX() &&
972 !
N->getOperand(1).hasOneUse();
975 N->getSimpleValueType(0).isVector() && !mayPreventLoadFold()) {
981 MVT VT =
N->getSimpleValueType(0);
989 CurDAG->getNode(NewOpcode,
DL, VT,
N->getOperand(0),
AllOnes);
991 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
998 switch (
N->getOpcode()) {
1000 MVT VT =
N->getSimpleValueType(0);
1002 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1009 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1012 CurDAG->getIntPtrConstant(
Index, dl));
1015 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1024 MVT VT =
N->getSimpleValueType(0);
1026 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1028 auto *MemNode = cast<MemSDNode>(
N);
1030 SDVTList VTs = CurDAG->getVTList(NarrowVT, MVT::Other);
1031 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()};
1032 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode(
1034 MemNode->getMemOperand());
1037 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1040 CurDAG->getIntPtrConstant(
Index, dl));
1044 CurDAG->ReplaceAllUsesWith(
N, To);
1055 auto *Ld = cast<LoadSDNode>(
N);
1056 MVT VT =
N->getSimpleValueType(0);
1064 SDValue Chain = Ld->getChain();
1066 auto *UserLd = dyn_cast<LoadSDNode>(
User);
1067 MVT UserVT =
User->getSimpleValueType(0);
1069 UserLd->getBasePtr() ==
Ptr && UserLd->getChain() == Chain &&
1070 !
User->hasAnyUseOfValue(1) &&
1084 CurDAG->getIntPtrConstant(0, dl));
1085 SDValue Res = CurDAG->getBitcast(VT, Extract);
1089 CurDAG->ReplaceAllUsesWith(
N, To);
1098 EVT EleVT =
N->getOperand(0).getValueType().getVectorElementType();
1099 if (EleVT == MVT::i1)
1102 assert(Subtarget->hasSSE41() &&
"Expected SSE4.1 support!");
1103 assert(
N->getValueType(0).getVectorElementType() != MVT::i16 &&
1104 "We can't replace VSELECT with BLENDV in vXi16!");
1106 if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(
N->getOperand(0)) ==
1109 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
1110 CurDAG->getTargetConstant(0xCA,
SDLoc(
N), MVT::i8));
1113 N->getOperand(0),
N->getOperand(1),
1117 CurDAG->ReplaceAllUsesWith(
N,
R.getNode());
1130 if (!
N->getSimpleValueType(0).isVector())
1134 switch (
N->getOpcode()) {
1144 if (
N->isStrictFPOpcode())
1146 CurDAG->getNode(NewOpc,
SDLoc(
N), {
N->getValueType(0), MVT::Other},
1147 {
N->getOperand(0),
N->getOperand(1)});
1150 CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1153 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1163 if (!
N->getValueType(0).isVector())
1167 switch (
N->getOpcode()) {
1173 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1174 N->getOperand(0),
N->getOperand(1));
1176 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1185 if (!
N->getValueType(0).isVector())
1189 if (
N->getOperand(0).getScalarValueSizeInBits() == 1) {
1191 "Unexpected opcode for mask vector!");
1199 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1202 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1222 switch (
N->getOpcode()) {
1238 bool IsStrict =
N->isStrictFPOpcode();
1242 {
N->getValueType(0), MVT::Other},
1243 {
N->getOperand(0),
N->getOperand(1),
1244 CurDAG->getTargetConstant(Imm, dl, MVT::i32)});
1248 CurDAG->getTargetConstant(Imm, dl, MVT::i32));
1250 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1261 MVT VT =
N->getSimpleValueType(0);
1262 if (VT.
isVector() || VT == MVT::f128)
1265 MVT VecVT = VT == MVT::f64 ? MVT::v2f64
1266 : VT == MVT::f32 ? MVT::v4f32
1276 if (Subtarget->hasSSE2()) {
1281 switch (
N->getOpcode()) {
1288 Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1);
1291 Res = CurDAG->getNode(
N->getOpcode(), dl, VecVT, Op0, Op1);
1294 CurDAG->getIntPtrConstant(0, dl));
1296 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1303 if (OptLevel != CodeGenOptLevel::None &&
1306 !Subtarget->useIndirectThunkCalls() &&
1307 ((
N->getOpcode() ==
X86ISD::CALL && !Subtarget->slowTwoMemOps()) ||
1309 (Subtarget->is64Bit() ||
1310 !getTargetMachine().isPositionIndependent())))) {
1349 switch (
N->getOpcode()) {
1354 MVT SrcVT =
N->getOperand(0).getSimpleValueType();
1355 MVT DstVT =
N->getSimpleValueType(0);
1367 if (SrcIsSSE && DstIsSSE)
1370 if (!SrcIsSSE && !DstIsSSE) {
1375 if (
N->getConstantOperandVal(1))
1383 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1384 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1392 CurDAG->getEntryNode(), dl,
N->getOperand(0), MemTmp, MPI, MemVT);
1394 MemTmp, MPI, MemVT);
1401 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Result);
1410 MVT SrcVT =
N->getOperand(1).getSimpleValueType();
1411 MVT DstVT =
N->getSimpleValueType(0);
1423 if (SrcIsSSE && DstIsSSE)
1426 if (!SrcIsSSE && !DstIsSSE) {
1431 if (
N->getConstantOperandVal(2))
1439 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1440 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1450 SDVTList VTs = CurDAG->getVTList(MVT::Other);
1451 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), MemTmp};
1455 if (
N->getFlags().hasNoFPExcept()) {
1457 Flags.setNoFPExcept(
true);
1458 Store->setFlags(Flags);
1461 assert(SrcVT == MemVT &&
"Unexpected VT!");
1462 Store = CurDAG->getStore(
N->getOperand(0), dl,
N->getOperand(1), MemTmp,
1467 SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other);
1469 Result = CurDAG->getMemIntrinsicNode(
1472 if (
N->getFlags().hasNoFPExcept()) {
1474 Flags.setNoFPExcept(
true);
1478 assert(DstVT == MemVT &&
"Unexpected VT!");
1479 Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI);
1487 CurDAG->ReplaceAllUsesWith(
N,
Result.getNode());
1501 CurDAG->RemoveDeadNodes();
1505bool X86DAGToDAGISel::tryOptimizeRem8Extend(
SDNode *
N) {
1506 unsigned Opc =
N->getMachineOpcode();
1507 if (Opc != X86::MOVZX32rr8 && Opc != X86::MOVSX32rr8 &&
1508 Opc != X86::MOVSX64rr8)
1520 unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX
1521 : X86::MOVSX32rr8_NOREX;
1526 if (Opc == X86::MOVSX64rr8) {
1531 ReplaceUses(
N, Extend);
1540void X86DAGToDAGISel::PostprocessISelDAG() {
1542 if (
TM.getOptLevel() == CodeGenOptLevel::None)
1547 bool MadeChange =
false;
1548 while (Position != CurDAG->allnodes_begin()) {
1551 if (
N->use_empty() || !
N->isMachineOpcode())
1554 if (tryOptimizeRem8Extend(
N)) {
1559 unsigned Opc =
N->getMachineOpcode();
1570 case X86::CTEST16rr:
1571 case X86::CTEST32rr:
1572 case X86::CTEST64rr: {
1573 auto &Op0 =
N->getOperand(0);
1578#define CASE_ND(OP) \
1581 switch (
And.getMachineOpcode()) {
1588 if (
And->hasAnyUseOfValue(1))
1591 Ops[0] =
And.getOperand(0);
1592 Ops[1] =
And.getOperand(1);
1594 CurDAG->getMachineNode(Opc,
SDLoc(
N), MVT::i32, Ops);
1595 ReplaceUses(
N,
Test);
1603 if (
And->hasAnyUseOfValue(1))
1606 bool IsCTESTCC = X86::isCTESTCC(Opc);
1607#define FROM_TO(A, B) \
1608 CASE_ND(A) NewOpc = IsCTESTCC ? X86::C##B : X86::B; \
1610 switch (
And.getMachineOpcode()) {
1620 And.getOperand(3),
And.getOperand(4),
1621 And.getOperand(5),
And.getOperand(0)};
1634 NewOpc,
SDLoc(
N), MVT::i32, MVT::Other, Ops);
1635 CurDAG->setNodeMemRefs(
1636 Test, cast<MachineSDNode>(
And.getNode())->memoperands());
1648 case X86::KORTESTBrr:
1649 case X86::KORTESTWrr:
1650 case X86::KORTESTDrr:
1651 case X86::KORTESTQrr: {
1653 if (Op0 !=
N->getOperand(1) || !
N->isOnlyUserOf(Op0.
getNode()) ||
1668#define FROM_TO(A, B) \
1680 if (NewOpc == X86::KTESTWrr && !Subtarget->hasDQI())
1685 ReplaceUses(
N, KTest);
1690 case TargetOpcode::SUBREG_TO_REG: {
1691 unsigned SubRegIdx =
N->getConstantOperandVal(2);
1692 if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm)
1709 CASE(VMOVAPDZ128rr)
CASE(VMOVUPDZ128rr)
1710 CASE(VMOVAPSZ128rr)
CASE(VMOVUPSZ128rr)
1711 CASE(VMOVDQA32Z128rr)
CASE(VMOVDQU32Z128rr)
1712 CASE(VMOVDQA64Z128rr)
CASE(VMOVDQU64Z128rr)
1713 CASE(VMOVAPDZ256rr)
CASE(VMOVUPDZ256rr)
1714 CASE(VMOVAPSZ256rr)
CASE(VMOVUPSZ256rr)
1715 CASE(VMOVDQA32Z256rr)
CASE(VMOVDQU32Z256rr)
1716 CASE(VMOVDQA64Z256rr)
CASE(VMOVDQU64Z256rr)
1721 if (!
In.isMachineOpcode() ||
1722 In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END)
1727 uint64_t TSFlags = getInstrInfo()->get(
In.getMachineOpcode()).TSFlags;
1735 CurDAG->UpdateNodeOperands(
N,
N->getOperand(0), In,
N->getOperand(2));
1742 CurDAG->RemoveDeadNodes();
1747void X86DAGToDAGISel::emitSpecialCodeForMain() {
1748 if (Subtarget->isTargetCygMing()) {
1750 auto &
DL = CurDAG->getDataLayout();
1753 CLI.setChain(CurDAG->getRoot())
1755 CurDAG->getExternalSymbol(
"__main", TLI->getPointerTy(
DL)),
1759 CurDAG->setRoot(
Result.second);
1763void X86DAGToDAGISel::emitFunctionEntryCode() {
1766 if (
F.hasExternalLinkage() &&
F.getName() ==
"main")
1767 emitSpecialCodeForMain();
1777 return isInt<31>(Val);
1781 X86ISelAddressMode &AM) {
1786 int64_t Val = AM.Disp +
Offset;
1789 if (Val != 0 && (AM.ES || AM.MCSym))
1793 if (Subtarget->is64Bit()) {
1796 AM.hasSymbolicDisplacement()))
1800 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
1819 if (Subtarget->isTarget64BitILP32() && !isUInt<31>(Val) &&
1820 !AM.hasBaseOrIndexReg())
1827bool X86DAGToDAGISel::matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
1828 bool AllowSegmentRegForX32) {
1840 if (
isNullConstant(Address) && AM.Segment.getNode() ==
nullptr &&
1841 !IndirectTlsSegRefs &&
1842 (Subtarget->isTargetGlibc() || Subtarget->isTargetAndroid() ||
1843 Subtarget->isTargetFuchsia())) {
1844 if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32)
1846 switch (
N->getPointerInfo().getAddrSpace()) {
1848 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
1851 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
1864bool X86DAGToDAGISel::matchWrapper(
SDValue N, X86ISelAddressMode &AM) {
1867 if (AM.hasSymbolicDisplacement())
1870 bool IsRIPRelTLS =
false;
1888 if (IsRIPRel && AM.hasBaseOrIndexReg())
1892 X86ISelAddressMode Backup = AM;
1896 if (
auto *
G = dyn_cast<GlobalAddressSDNode>(N0)) {
1897 AM.GV =
G->getGlobal();
1898 AM.SymbolFlags =
G->getTargetFlags();
1900 }
else if (
auto *CP = dyn_cast<ConstantPoolSDNode>(N0)) {
1901 AM.CP =
CP->getConstVal();
1902 AM.Alignment =
CP->getAlign();
1903 AM.SymbolFlags =
CP->getTargetFlags();
1905 }
else if (
auto *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
1906 AM.ES = S->getSymbol();
1907 AM.SymbolFlags = S->getTargetFlags();
1908 }
else if (
auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
1909 AM.MCSym = S->getMCSymbol();
1910 }
else if (
auto *J = dyn_cast<JumpTableSDNode>(N0)) {
1911 AM.JT = J->getIndex();
1912 AM.SymbolFlags = J->getTargetFlags();
1913 }
else if (
auto *BA = dyn_cast<BlockAddressSDNode>(N0)) {
1914 AM.BlockAddr = BA->getBlockAddress();
1915 AM.SymbolFlags = BA->getTargetFlags();
1916 Offset = BA->getOffset();
1921 if (Subtarget->is64Bit() && !IsRIPRel && AM.GV &&
1922 TM.isLargeGlobalValue(AM.GV)) {
1927 if (foldOffsetIntoAddress(
Offset, AM)) {
1933 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64));
1941bool X86DAGToDAGISel::matchAddress(
SDValue N, X86ISelAddressMode &AM) {
1942 if (matchAddressRecursively(
N, AM, 0))
1949 if (Subtarget->isTarget64BitILP32() &&
1950 AM.BaseType == X86ISelAddressMode::RegBase &&
1951 AM.Base_Reg.getNode() !=
nullptr && AM.IndexReg.getNode() ==
nullptr) {
1952 SDValue Save_Base_Reg = AM.Base_Reg;
1953 if (
auto *LoadN = dyn_cast<LoadSDNode>(Save_Base_Reg)) {
1955 if (matchLoadInAddress(LoadN, AM,
true))
1956 AM.Base_Reg = Save_Base_Reg;
1962 if (AM.Scale == 2 &&
1963 AM.BaseType == X86ISelAddressMode::RegBase &&
1964 AM.Base_Reg.getNode() ==
nullptr) {
1965 AM.Base_Reg = AM.IndexReg;
1972 (!AM.GV || !
TM.isLargeGlobalValue(AM.GV)) && Subtarget->is64Bit() &&
1973 AM.Scale == 1 && AM.BaseType == X86ISelAddressMode::RegBase &&
1974 AM.Base_Reg.getNode() ==
nullptr && AM.IndexReg.getNode() ==
nullptr &&
1976 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64);
1982bool X86DAGToDAGISel::matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
1988 X86ISelAddressMode Backup = AM;
1989 if (!matchAddressRecursively(
N.getOperand(0), AM,
Depth+1) &&
1990 !matchAddressRecursively(Handle.getValue().getOperand(1), AM,
Depth+1))
1995 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM,
1997 !matchAddressRecursively(Handle.getValue().getOperand(0), AM,
Depth + 1))
2004 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2005 !AM.Base_Reg.getNode() &&
2006 !AM.IndexReg.getNode()) {
2007 N = Handle.getValue();
2008 AM.Base_Reg =
N.getOperand(0);
2009 AM.IndexReg =
N.getOperand(1);
2013 N = Handle.getValue();
2023 if (
N->getNodeId() == -1 ||
2043 X86ISelAddressMode &AM) {
2050 if (ScaleLog <= 0 || ScaleLog >= 4 ||
2051 Mask != (0xffu << ScaleLog))
2054 MVT XVT =
X.getSimpleValueType();
2055 MVT VT =
N.getSimpleValueType();
2080 AM.Scale = (1 << ScaleLog);
2088 X86ISelAddressMode &AM) {
2094 int64_t Mask = cast<ConstantSDNode>(
N->getOperand(1))->getSExtValue();
2099 bool FoundAnyExtend =
false;
2103 FoundAnyExtend =
true;
2121 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3)
2124 MVT VT =
N.getSimpleValueType();
2126 if (FoundAnyExtend) {
2147 AM.Scale = 1 << ShiftAmt;
2148 AM.IndexReg = NewAnd;
2182 X86ISelAddressMode &AM) {
2188 unsigned MaskIdx, MaskLen;
2191 unsigned MaskLZ = 64 - (MaskIdx + MaskLen);
2197 unsigned AMShiftAmt = MaskIdx;
2201 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2205 unsigned ScaleDown = (64 -
X.getSimpleValueType().getSizeInBits()) + ShiftAmt;
2206 if (MaskLZ < ScaleDown)
2208 MaskLZ -= ScaleDown;
2216 bool ReplacingAnyExtend =
false;
2218 unsigned ExtendBits =
X.getSimpleValueType().getSizeInBits() -
2219 X.getOperand(0).getSimpleValueType().getSizeInBits();
2222 X =
X.getOperand(0);
2223 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits;
2224 ReplacingAnyExtend =
true;
2226 APInt MaskedHighBits =
2233 MVT VT =
N.getSimpleValueType();
2234 if (ReplacingAnyExtend) {
2235 assert(
X.getValueType() != VT);
2242 MVT XVT =
X.getSimpleValueType();
2263 AM.Scale = 1 << AMShiftAmt;
2264 AM.IndexReg = NewExt;
2274 X86ISelAddressMode &AM,
2282 if (!Subtarget.hasTBM() &&
2283 !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR()))
2287 unsigned MaskIdx, MaskLen;
2295 unsigned AMShiftAmt = MaskIdx;
2299 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2301 MVT XVT =
X.getSimpleValueType();
2302 MVT VT =
N.getSimpleValueType();
2327 AM.Scale = 1 << AMShiftAmt;
2328 AM.IndexReg = NewExt;
2335 X86ISelAddressMode &AM,
2337 assert(AM.IndexReg.getNode() ==
nullptr &&
"IndexReg already matched");
2338 assert((AM.Scale == 1 || AM.Scale == 2 || AM.Scale == 4 || AM.Scale == 8) &&
2339 "Illegal index scale");
2345 EVT VT =
N.getValueType();
2346 unsigned Opc =
N.getOpcode();
2349 if (CurDAG->isBaseWithConstantOffset(
N)) {
2350 auto *AddVal = cast<ConstantSDNode>(
N.getOperand(1));
2352 if (!foldOffsetIntoAddress(
Offset, AM))
2353 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2357 if (Opc ==
ISD::ADD &&
N.getOperand(0) ==
N.getOperand(1)) {
2358 if (AM.Scale <= 4) {
2360 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2366 uint64_t ShiftAmt =
N.getConstantOperandVal(1);
2367 uint64_t ScaleAmt = 1ULL << ShiftAmt;
2368 if ((AM.Scale * ScaleAmt) <= 8) {
2369 AM.Scale *= ScaleAmt;
2370 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2378 if (Src.getOpcode() ==
ISD::ADD && Src->getFlags().hasNoSignedWrap() &&
2380 if (CurDAG->isBaseWithConstantOffset(Src)) {
2381 SDValue AddSrc = Src.getOperand(0);
2382 auto *AddVal = cast<ConstantSDNode>(Src.getOperand(1));
2384 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2386 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2392 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2393 CurDAG->RemoveDeadNode(
N.getNode());
2405 unsigned SrcOpc = Src.getOpcode();
2406 if (((SrcOpc ==
ISD::ADD && Src->getFlags().hasNoUnsignedWrap()) ||
2407 CurDAG->isADDLike(Src,
true)) &&
2409 if (CurDAG->isBaseWithConstantOffset(Src)) {
2410 SDValue AddSrc = Src.getOperand(0);
2412 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2423 if ((AM.Scale * ScaleAmt) <= 8 &&
2425 CurDAG->MaskedValueIsZero(ShVal, HiBits))) {
2426 AM.Scale *= ScaleAmt;
2427 SDValue ExtShVal = CurDAG->getNode(Opc,
DL, VT, ShVal);
2436 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2438 SDValue ExtAdd = CurDAG->getNode(SrcOpc,
DL, VT, ExtSrc, ExtVal);
2442 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2443 CurDAG->RemoveDeadNode(
N.getNode());
2444 return Res ? Res : ExtSrc;
2454bool X86DAGToDAGISel::matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
2458 dbgs() <<
"MatchAddress: ";
2463 return matchAddressBase(
N, AM);
2468 if (AM.isRIPRelative()) {
2472 if (!(AM.ES || AM.MCSym) && AM.JT != -1)
2475 if (
auto *Cst = dyn_cast<ConstantSDNode>(
N))
2476 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM))
2481 switch (
N.getOpcode()) {
2484 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
2485 if (
const auto *ESNode = dyn_cast<MCSymbolSDNode>(
N.getOperand(0))) {
2487 AM.MCSym = ESNode->getMCSymbol();
2493 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2494 if (!foldOffsetIntoAddress(Val, AM))
2501 if (!matchWrapper(
N, AM))
2506 if (!matchLoadInAddress(cast<LoadSDNode>(
N), AM))
2511 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2512 AM.Base_Reg.getNode() ==
nullptr &&
2514 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
2515 AM.Base_FrameIndex = cast<FrameIndexSDNode>(
N)->getIndex();
2521 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2524 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
2525 unsigned Val = CN->getZExtValue();
2530 if (Val == 1 || Val == 2 || Val == 3) {
2532 AM.Scale = 1 << Val;
2533 AM.IndexReg = matchIndexRecursively(ShVal, AM,
Depth + 1);
2541 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2545 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2546 "Unexpected value size!");
2555 if (!isa<ConstantSDNode>(
N.getOperand(1)) ||
2556 !isa<ConstantSDNode>(
And.getOperand(1)))
2558 uint64_t Mask =
And.getConstantOperandVal(1) >>
N.getConstantOperandVal(1);
2570 if (
N.getResNo() != 0)
break;
2575 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2576 AM.Base_Reg.getNode() ==
nullptr &&
2577 AM.IndexReg.getNode() ==
nullptr) {
2578 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
2579 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
2580 CN->getZExtValue() == 9) {
2581 AM.Scale =
unsigned(CN->getZExtValue())-1;
2592 auto *AddVal = cast<ConstantSDNode>(MulVal.
getOperand(1));
2593 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue();
2594 if (foldOffsetIntoAddress(Disp, AM))
2595 Reg =
N.getOperand(0);
2597 Reg =
N.getOperand(0);
2600 AM.IndexReg = AM.Base_Reg =
Reg;
2619 X86ISelAddressMode Backup = AM;
2620 if (matchAddressRecursively(
N.getOperand(0), AM,
Depth+1)) {
2621 N = Handle.getValue();
2625 N = Handle.getValue();
2627 if (AM.IndexReg.getNode() || AM.isRIPRelative()) {
2642 RHS.getOperand(0).getValueType() == MVT::i32))
2646 if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() &&
2647 !AM.Base_Reg.getNode()->hasOneUse()) ||
2648 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
2652 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
2653 ((AM.Disp != 0) && (Backup.Disp == 0)) +
2654 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
2666 AM.NegateIndex =
true;
2674 if (!CurDAG->isADDLike(
N))
2678 if (!matchAdd(
N, AM,
Depth))
2687 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2691 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2692 "Unexpected value size!");
2694 if (!isa<ConstantSDNode>(
N.getOperand(1)))
2697 if (
N.getOperand(0).getOpcode() ==
ISD::SRL) {
2726 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2736 AM.IndexReg =
Index;
2742 if (Src.getOpcode() ==
ISD::AND && Src.hasOneUse())
2743 if (
auto *MaskC = dyn_cast<ConstantSDNode>(Src.getOperand(1))) {
2744 Mask = MaskC->getAPIntValue();
2745 Src = Src.getOperand(0);
2748 if (Src.getOpcode() ==
ISD::SHL && Src.hasOneUse() &&
N->hasOneUse()) {
2750 SDValue ShlSrc = Src.getOperand(0);
2751 SDValue ShlAmt = Src.getOperand(1);
2752 auto *ShAmtC = dyn_cast<ConstantSDNode>(ShlAmt);
2755 unsigned ShAmtV = ShAmtC->getZExtValue();
2763 if (!Src->getFlags().hasNoUnsignedWrap() &&
2764 !CurDAG->MaskedValueIsZero(ShlSrc, HighZeros & Mask))
2772 MVT VT =
N.getSimpleValueType();
2776 if (!
Mask.isAllOnes()) {
2777 Res = CurDAG->getConstant(
Mask.lshr(ShAmtV),
DL, SrcVT);
2779 Res = CurDAG->getNode(
ISD::AND,
DL, SrcVT, ShlSrc, Res);
2786 CurDAG->ReplaceAllUsesWith(
N, NewShl);
2787 CurDAG->RemoveDeadNode(
N.getNode());
2790 AM.Scale = 1 << ShAmtV;
2794 AM.IndexReg = matchIndexRecursively(Zext, AM,
Depth + 1);
2798 if (Src.getOpcode() ==
ISD::SRL && !
Mask.isAllOnes()) {
2801 Src.getOperand(0), AM))
2806 Src.getOperand(0), AM))
2811 Src.getOperand(0), AM, *Subtarget))
2819 return matchAddressBase(
N, AM);
2824bool X86DAGToDAGISel::matchAddressBase(
SDValue N, X86ISelAddressMode &AM) {
2826 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
2828 if (!AM.IndexReg.getNode()) {
2839 AM.BaseType = X86ISelAddressMode::RegBase;
2844bool X86DAGToDAGISel::matchVectorAddressRecursively(
SDValue N,
2845 X86ISelAddressMode &AM,
2849 dbgs() <<
"MatchVectorAddress: ";
2854 return matchAddressBase(
N, AM);
2857 switch (
N.getOpcode()) {
2859 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2860 if (!foldOffsetIntoAddress(Val, AM))
2865 if (!matchWrapper(
N, AM))
2873 X86ISelAddressMode Backup = AM;
2874 if (!matchVectorAddressRecursively(
N.getOperand(0), AM,
Depth + 1) &&
2875 !matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2881 if (!matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2883 !matchVectorAddressRecursively(Handle.getValue().getOperand(0), AM,
2888 N = Handle.getValue();
2893 return matchAddressBase(
N, AM);
2899bool X86DAGToDAGISel::matchVectorAddress(
SDValue N, X86ISelAddressMode &AM) {
2900 return matchVectorAddressRecursively(
N, AM, 0);
2908 X86ISelAddressMode AM;
2914 AM.IndexReg = matchIndexRecursively(IndexOp, AM, 0);
2916 AM.IndexReg = IndexOp;
2920 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2922 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2924 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2930 if (matchVectorAddress(BasePtr, AM))
2933 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2947 X86ISelAddressMode AM;
2959 unsigned AddrSpace =
2960 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
2962 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2964 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2966 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2971 MVT VT =
N.getSimpleValueType();
2973 if (matchAddress(
N, AM))
2976 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2992 N =
N.getOperand(0);
3007 const GlobalValue *GV = cast<GlobalAddressSDNode>(
N)->getGlobal();
3009 return CR->getUnsignedMax().ult(1ull << 32);
3011 return !
TM.isLargeGlobalValue(GV);
3020 if (!selectLEAAddr(
N,
Base, Scale,
Index, Disp, Segment))
3023 auto *
RN = dyn_cast<RegisterSDNode>(
Base);
3024 if (RN &&
RN->getReg() == 0)
3025 Base = CurDAG->getRegister(0, MVT::i64);
3026 else if (
Base.getValueType() == MVT::i32 && !isa<FrameIndexSDNode>(
Base)) {
3030 Base = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
3034 RN = dyn_cast<RegisterSDNode>(
Index);
3035 if (RN &&
RN->getReg() == 0)
3036 Index = CurDAG->getRegister(0, MVT::i64);
3039 "Expect to be extending 32-bit registers for use in LEA");
3042 Index = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
3051bool X86DAGToDAGISel::selectLEAAddr(
SDValue N,
3055 X86ISelAddressMode AM;
3059 MVT VT =
N.getSimpleValueType();
3064 SDValue T = CurDAG->getRegister(0, MVT::i32);
3066 if (matchAddress(
N, AM))
3071 unsigned Complexity = 0;
3072 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode())
3074 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
3077 if (AM.IndexReg.getNode())
3090 if (AM.hasSymbolicDisplacement()) {
3092 if (Subtarget->is64Bit())
3102 auto isMathWithFlags = [](
SDValue V) {
3103 switch (
V.getOpcode()) {
3124 if (isMathWithFlags(
N.getOperand(0)) || isMathWithFlags(
N.getOperand(1)))
3132 if (Complexity <= 2)
3135 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
3146 X86ISelAddressMode AM;
3147 if (
auto *GA = dyn_cast<GlobalAddressSDNode>(
N)) {
3148 AM.GV = GA->getGlobal();
3149 AM.Disp += GA->getOffset();
3150 AM.SymbolFlags = GA->getTargetFlags();
3152 auto *SA = cast<ExternalSymbolSDNode>(
N);
3153 AM.ES = SA->getSymbol();
3154 AM.SymbolFlags = SA->getTargetFlags();
3157 if (Subtarget->is32Bit()) {
3159 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
3162 MVT VT =
N.getSimpleValueType();
3163 getAddressOperands(AM,
SDLoc(
N), VT,
Base, Scale,
Index, Disp, Segment);
3171 EVT VT =
N.getValueType();
3172 bool WasTruncated =
false;
3174 WasTruncated =
true;
3175 N =
N.getOperand(0);
3184 unsigned Opc =
N.getOperand(0)->getOpcode();
3186 Op =
N.getOperand(0);
3189 return !WasTruncated;
3193 auto *GA = cast<GlobalAddressSDNode>(
N.getOperand(0));
3194 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange();
3195 if (!CR || CR->getUnsignedMax().uge(1ull << VT.
getSizeInBits()))
3199 Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(),
SDLoc(
N), VT,
3200 GA->getOffset(), GA->getTargetFlags());
3208 assert(Root &&
P &&
"Unknown root/parent nodes");
3210 !IsProfitableToFold(
N,
P, Root) ||
3211 !IsLegalToFold(
N,
P, Root, OptLevel))
3214 return selectAddr(
N.getNode(),
3215 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3222 assert(Root &&
P &&
"Unknown root/parent nodes");
3224 !IsProfitableToFold(
N,
P, Root) ||
3225 !IsLegalToFold(
N,
P, Root, OptLevel))
3228 return selectAddr(
N.getNode(),
3229 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3235SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
3236 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
3238 return CurDAG->getRegister(GlobalBaseReg, TLI->
getPointerTy(
DL)).getNode();
3241bool X86DAGToDAGISel::isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const {
3243 N =
N->getOperand(0).getNode();
3247 auto *GA = dyn_cast<GlobalAddressSDNode>(
N->getOperand(0));
3251 auto *GV = GA->getGlobal();
3254 return CR->getSignedMin().sge(-1ull << Width) &&
3255 CR->getSignedMax().slt(1ull << Width);
3260 return Width == 32 && !
TM.isLargeGlobalValue(GV);
3264 assert(
N->isMachineOpcode() &&
"Unexpected node");
3265 unsigned Opc =
N->getMachineOpcode();
3266 const MCInstrDesc &MCID = getInstrInfo()->get(Opc);
3271 return static_cast<X86::CondCode>(
N->getConstantOperandVal(CondNo));
3276bool X86DAGToDAGISel::onlyUsesZeroFlag(
SDValue Flags)
const {
3281 if (UI.getUse().getResNo() !=
Flags.getResNo())
3285 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3289 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3291 if (FlagUI.getUse().getResNo() != 1)
continue;
3293 if (!FlagUI->isMachineOpcode())
return false;
3312bool X86DAGToDAGISel::hasNoSignFlagUses(
SDValue Flags)
const {
3317 if (UI.getUse().getResNo() !=
Flags.getResNo())
3321 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3325 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3327 if (FlagUI.getUse().getResNo() != 1)
continue;
3329 if (!FlagUI->isMachineOpcode())
return false;
3368 bool X86DAGToDAGISel::hasNoCarryFlagUses(
SDValue Flags)
const {
3373 if (UI.getUse().getResNo() !=
Flags.getResNo())
3376 unsigned UIOpc = UI->getOpcode();
3380 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3384 FlagUI != FlagUE; ++FlagUI) {
3386 if (FlagUI.getUse().getResNo() != 1)
3389 if (!FlagUI->isMachineOpcode())
3430 if (StoredVal.
getResNo() != 0)
return false;
3444 LoadNode = cast<LoadSDNode>(Load);
3447 if (!Load.hasOneUse())
3455 bool FoundLoad =
false;
3459 const unsigned int Max = 1024;
3501 if (Chain == Load.getValue(1)) {
3507 if (
Op == Load.getValue(1)) {
3523 if (
Op.getNode() != LoadNode)
3556bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(
SDNode *
Node) {
3557 auto *StoreNode = cast<StoreSDNode>(
Node);
3564 EVT MemVT = StoreNode->getMemoryVT();
3565 if (MemVT != MVT::i64 && MemVT != MVT::i32 && MemVT != MVT::i16 &&
3569 bool IsCommutable =
false;
3570 bool IsNegate =
false;
3584 IsCommutable =
true;
3588 unsigned LoadOpNo = IsNegate ? 1 : 0;
3592 LoadNode, InputChain)) {
3599 LoadNode, InputChain))
3608 auto SelectOpcode = [&](
unsigned Opc64,
unsigned Opc32,
unsigned Opc16,
3629 unsigned NewOpc = SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m,
3639 if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) {
3643 if ((IsOne || IsNegOne) && hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3646 ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
3647 : SelectOpcode(
X86::DEC64m,
X86::DEC32m,
X86::DEC16m,
X86::DEC8m);
3660 auto SelectRegOpcode = [SelectOpcode](
unsigned Opc) {
3663 return SelectOpcode(X86::ADD64mr, X86::ADD32mr, X86::ADD16mr,
3666 return SelectOpcode(X86::ADC64mr, X86::ADC32mr, X86::ADC16mr,
3669 return SelectOpcode(X86::SUB64mr, X86::SUB32mr, X86::SUB16mr,
3672 return SelectOpcode(X86::SBB64mr, X86::SBB32mr, X86::SBB16mr,
3675 return SelectOpcode(X86::AND64mr, X86::AND32mr, X86::AND16mr,
3678 return SelectOpcode(X86::OR64mr, X86::OR32mr, X86::OR16mr, X86::OR8mr);
3680 return SelectOpcode(X86::XOR64mr, X86::XOR32mr, X86::XOR16mr,
3686 auto SelectImmOpcode = [SelectOpcode](
unsigned Opc) {
3689 return SelectOpcode(X86::ADD64mi32, X86::ADD32mi, X86::ADD16mi,
3692 return SelectOpcode(X86::ADC64mi32, X86::ADC32mi, X86::ADC16mi,
3695 return SelectOpcode(X86::SUB64mi32, X86::SUB32mi, X86::SUB16mi,
3698 return SelectOpcode(X86::SBB64mi32, X86::SBB32mi, X86::SBB16mi,
3701 return SelectOpcode(X86::AND64mi32, X86::AND32mi, X86::AND16mi,
3704 return SelectOpcode(X86::OR64mi32, X86::OR32mi, X86::OR16mi,
3707 return SelectOpcode(X86::XOR64mi32, X86::XOR32mi, X86::XOR16mi,
3714 unsigned NewOpc = SelectRegOpcode(Opc);
3719 if (
auto *OperandC = dyn_cast<ConstantSDNode>(Operand)) {
3720 int64_t OperandV = OperandC->getSExtValue();
3726 ((MemVT != MVT::i8 && !isInt<8>(OperandV) && isInt<8>(-OperandV)) ||
3727 (MemVT == MVT::i64 && !isInt<32>(OperandV) &&
3728 isInt<32>(-OperandV))) &&
3729 hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3730 OperandV = -OperandV;
3734 if (MemVT != MVT::i64 || isInt<32>(OperandV)) {
3735 Operand = CurDAG->getTargetConstant(OperandV,
SDLoc(
Node), MemVT);
3736 NewOpc = SelectImmOpcode(Opc);
3742 CurDAG->getCopyToReg(InputChain,
SDLoc(
Node), X86::EFLAGS,
3746 Segment, Operand, CopyTo, CopyTo.
getValue(1)};
3747 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3751 Segment, Operand, InputChain};
3752 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3763 CurDAG->setNodeMemRefs(Result, MemOps);
3769 CurDAG->RemoveDeadNode(
Node);
3780bool X86DAGToDAGISel::matchBitExtract(
SDNode *
Node) {
3784 "Should be either an and-mask, or right-shift after clearing high bits.");
3787 if (!Subtarget->hasBMI() && !Subtarget->hasBMI2())
3790 MVT NVT =
Node->getSimpleValueType(0);
3793 if (NVT != MVT::i32 && NVT != MVT::i64)
3801 const bool AllowExtraUsesByDefault = Subtarget->hasBMI2();
3802 auto checkUses = [AllowExtraUsesByDefault](
3804 std::optional<bool> AllowExtraUses) {
3805 return AllowExtraUses.value_or(AllowExtraUsesByDefault) ||
3806 Op.getNode()->hasNUsesOfValue(NUses,
Op.getResNo());
3808 auto checkOneUse = [checkUses](
SDValue Op,
3809 std::optional<bool> AllowExtraUses =
3811 return checkUses(
Op, 1, AllowExtraUses);
3813 auto checkTwoUse = [checkUses](
SDValue Op,
3814 std::optional<bool> AllowExtraUses =
3816 return checkUses(
Op, 2, AllowExtraUses);
3819 auto peekThroughOneUseTruncation = [checkOneUse](
SDValue V) {
3821 assert(
V.getSimpleValueType() == MVT::i32 &&
3822 V.getOperand(0).getSimpleValueType() == MVT::i64 &&
3823 "Expected i64 -> i32 truncation");
3824 V =
V.getOperand(0);
3830 auto matchPatternA = [checkOneUse, peekThroughOneUseTruncation, &NBits,
3833 if (
Mask->getOpcode() !=
ISD::ADD || !checkOneUse(Mask))
3839 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3844 NBits =
M0->getOperand(1);
3845 NegateNBits =
false;
3849 auto isAllOnes = [
this, peekThroughOneUseTruncation, NVT](
SDValue V) {
3850 V = peekThroughOneUseTruncation(V);
3851 return CurDAG->MaskedValueIsAllOnes(
3857 auto matchPatternB = [checkOneUse, isAllOnes, peekThroughOneUseTruncation,
3860 if (
Mask.getOpcode() !=
ISD::XOR || !checkOneUse(Mask))
3863 if (!isAllOnes(
Mask->getOperand(1)))
3866 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3870 if (!isAllOnes(
M0->getOperand(0)))
3872 NBits =
M0->getOperand(1);
3873 NegateNBits =
false;
3879 auto canonicalizeShiftAmt = [&NBits, &NegateNBits](
SDValue ShiftAmt,
3880 unsigned Bitwidth) {
3885 NBits = NBits.getOperand(0);
3890 auto *V0 = dyn_cast<ConstantSDNode>(NBits.getOperand(0));
3891 if (!V0 || V0->getZExtValue() != Bitwidth)
3893 NBits = NBits.getOperand(1);
3894 NegateNBits =
false;
3900 auto matchPatternC = [checkOneUse, peekThroughOneUseTruncation, &NegateNBits,
3903 Mask = peekThroughOneUseTruncation(Mask);
3904 unsigned Bitwidth =
Mask.getSimpleValueType().getSizeInBits();
3906 if (
Mask.getOpcode() !=
ISD::SRL || !checkOneUse(Mask))
3913 if (!checkOneUse(
M1))
3915 canonicalizeShiftAmt(
M1, Bitwidth);
3920 return !NegateNBits;
3928 auto matchPatternD = [checkOneUse, checkTwoUse, canonicalizeShiftAmt,
3929 AllowExtraUsesByDefault, &NegateNBits,
3942 canonicalizeShiftAmt(N1, Bitwidth);
3946 const bool AllowExtraUses = AllowExtraUsesByDefault && !NegateNBits;
3947 if (!checkOneUse(N0, AllowExtraUses) || !checkTwoUse(N1, AllowExtraUses))
3953 auto matchLowBitMask = [matchPatternA, matchPatternB,
3955 return matchPatternA(Mask) || matchPatternB(Mask) || matchPatternC(Mask);
3959 X =
Node->getOperand(0);
3962 if (matchLowBitMask(Mask)) {
3966 if (!matchLowBitMask(Mask))
3970 X = CurDAG->getAllOnesConstant(
SDLoc(
Node), NVT);
3971 }
else if (!matchPatternD(
Node))
3976 if (NegateNBits && !Subtarget->hasBMI2())
3988 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, MVT::i32), 0);
3991 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit,
DL, MVT::i32);
3993 NBits =
SDValue(CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG,
DL,
3994 MVT::i32, ImplDef, NBits, SRIdxVal),
4004 NBits = CurDAG->getNode(
ISD::SUB,
DL, MVT::i32, BitWidthC, NBits);
4008 if (Subtarget->hasBMI2()) {
4010 if (NVT != MVT::i32) {
4018 SelectCode(Extract.
getNode());
4027 SDValue RealX = peekThroughOneUseTruncation(
X);
4033 MVT XVT =
X.getSimpleValueType();
4043 SDValue C8 = CurDAG->getConstant(8,
DL, MVT::i8);
4051 SDValue ShiftAmt =
X.getOperand(1);
4052 X =
X.getOperand(0);
4055 "Expected shift amount to be i8");
4059 SDValue OrigShiftAmt = ShiftAmt;
4064 Control = CurDAG->getNode(
ISD::OR,
DL, MVT::i32, Control, ShiftAmt);
4069 if (XVT != MVT::i32) {
4084 SelectCode(Extract.
getNode());
4091 MVT NVT =
Node->getSimpleValueType(0);
4104 Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR());
4105 if (!PreferBEXTR && !Subtarget->hasBMI2())
4117 if (NVT != MVT::i32 && NVT != MVT::i64)
4121 auto *MaskCst = dyn_cast<ConstantSDNode>(N1);
4122 auto *ShiftCst = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
4123 if (!MaskCst || !ShiftCst)
4131 uint64_t Shift = ShiftCst->getZExtValue();
4136 if (Shift == 8 && MaskSize == 8)
4147 if (!PreferBEXTR && MaskSize <= 32)
4151 unsigned ROpc, MOpc;
4153#define GET_EGPR_IF_ENABLED(OPC) (Subtarget->hasEGPR() ? OPC##_EVEX : OPC)
4155 assert(Subtarget->hasBMI2() &&
"We must have BMI2's BZHI then.");
4159 Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT);
4164 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4165 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4171 Control = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT);
4172 if (Subtarget->hasTBM()) {
4173 ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri;
4174 MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi;
4176 assert(Subtarget->hasBMI() &&
"We must have BMI1's BEXTR then.");
4182 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4183 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4189 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4190 if (tryFoldLoad(
Node, N0.
getNode(), Input, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4192 Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Control, Input.
getOperand(0)};
4193 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
4194 NewNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4198 CurDAG->setNodeMemRefs(NewNode, {cast<LoadSDNode>(Input)->getMemOperand()});
4200 NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, Control);
4205 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT);
4209 CurDAG->getMachineNode(NewOpc, dl, NVT,
SDValue(NewNode, 0), ShAmt);
4216MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
4217 bool MayFoldLoad,
const SDLoc &dl,
4222 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4223 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4226 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4227 if (MayFoldLoad && tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4228 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4230 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other);
4231 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4235 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
4240 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32);
4241 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4248MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
4249 bool MayFoldLoad,
const SDLoc &dl,
4255 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4256 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4259 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4260 if (MayFoldLoad && tryFoldLoad(
Node, N2, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4261 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4263 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other, MVT::Glue);
4264 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4269 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N2)->getMemOperand()});
4274 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Glue);
4275 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4280bool X86DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
4281 EVT VT =
N->getValueType(0);
4288 unsigned Size = VT == MVT::i64 ? 64 : 32;
4290 SDValue OrigShiftAmt =
N->getOperand(1);
4291 SDValue ShiftAmt = OrigShiftAmt;
4306 auto *Add0C = dyn_cast<ConstantSDNode>(Add0);
4307 auto *Add1C = dyn_cast<ConstantSDNode>(Add1);
4310 if (Add1C && Add1C->getAPIntValue().urem(
Size) == 0) {
4314 ((Add0C && Add0C->getAPIntValue().urem(
Size) ==
Size - 1) ||
4315 (Add1C && Add1C->getAPIntValue().urem(
Size) ==
Size - 1))) {
4319 assert(Add0C ==
nullptr || Add1C ==
nullptr);
4328 NewShiftAmt = CurDAG->getNode(
ISD::XOR,
DL, OpVT,
4329 Add0C ==
nullptr ? Add0 : Add1,
AllOnes);
4335 Add0C->getZExtValue() != 0) {
4338 if (Add0C->getZExtValue() %
Size == 0)
4341 Add0C->getZExtValue() % 32 == 0) {
4349 Add0 = CurDAG->getZExtOrTrunc(Add0,
DL, SubVT);
4353 X = CurDAG->getNode(
ISD::ADD,
DL, SubVT, Add1, Add0);
4375 NewShiftAmt = CurDAG->getNode(
ISD::TRUNCATE,
DL, MVT::i8, NewShiftAmt);
4382 NewShiftAmt = CurDAG->getNode(
ISD::AND,
DL, MVT::i8, NewShiftAmt,
4383 CurDAG->getConstant(
Size - 1,
DL, MVT::i8));
4387 SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(
N,
N->getOperand(0),
4389 if (UpdatedNode !=
N) {
4392 ReplaceNode(
N, UpdatedNode);
4399 CurDAG->RemoveDeadNode(OrigShiftAmt.
getNode());
4407bool X86DAGToDAGISel::tryShrinkShlLogicImm(
SDNode *
N) {
4408 MVT NVT =
N->getSimpleValueType(0);
4409 unsigned Opcode =
N->getOpcode();
4417 auto *Cst = dyn_cast<ConstantSDNode>(N1);
4421 int64_t Val = Cst->getSExtValue();
4426 bool FoundAnyExtend =
false;
4430 FoundAnyExtend =
true;
4438 if (NVT != MVT::i32 && NVT != MVT::i64)
4441 auto *ShlCst = dyn_cast<ConstantSDNode>(Shift.
getOperand(1));
4445 uint64_t ShAmt = ShlCst->getZExtValue();
4449 uint64_t RemovedBitsMask = (1ULL << ShAmt) - 1;
4450 if (Opcode !=
ISD::AND && (Val & RemovedBitsMask) != 0)
4455 auto CanShrinkImmediate = [&](int64_t &ShiftedVal) {
4459 ShiftedVal = (
uint64_t)Val >> ShAmt;
4460 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4463 if (ShiftedVal == UINT8_MAX || ShiftedVal == UINT16_MAX)
4466 ShiftedVal = Val >> ShAmt;
4467 if ((!isInt<8>(Val) && isInt<8>(ShiftedVal)) ||
4468 (!isInt<32>(Val) && isInt<32>(ShiftedVal)))
4472 ShiftedVal = (
uint64_t)Val >> ShAmt;
4473 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4480 if (!CanShrinkImmediate(ShiftedVal))
4490 unsigned ZExtWidth = Cst->getAPIntValue().getActiveBits();
4496 NeededMask &= ~Cst->getAPIntValue();
4498 if (CurDAG->MaskedValueIsZero(
N->getOperand(0), NeededMask))
4503 if (FoundAnyExtend) {
4509 SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT);
4511 SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT,
X, NewCst);
4520bool X86DAGToDAGISel::matchVPTERNLOG(
SDNode *Root,
SDNode *ParentA,
4524 assert(
A.isOperandOf(ParentA) &&
B.isOperandOf(ParentB) &&
4525 C.isOperandOf(ParentC) &&
"Incorrect parent node");
4527 auto tryFoldLoadOrBCast =
4530 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4536 L =
L.getOperand(0);
4543 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4544 unsigned Size = MemIntr->getMemoryVT().getSizeInBits();
4548 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4551 bool FoldedLoad =
false;
4552 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4553 if (tryFoldLoadOrBCast(Root, ParentC,
C, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4555 }
else if (tryFoldLoadOrBCast(Root, ParentA,
A, Tmp0, Tmp1, Tmp2, Tmp3,
4560 uint8_t OldImm =
Imm;
4561 Imm = OldImm & 0xa5;
4562 if (OldImm & 0x02)
Imm |= 0x10;
4563 if (OldImm & 0x10)
Imm |= 0x02;
4564 if (OldImm & 0x08)
Imm |= 0x40;
4565 if (OldImm & 0x40)
Imm |= 0x08;
4566 }
else if (tryFoldLoadOrBCast(Root, ParentB,
B, Tmp0, Tmp1, Tmp2, Tmp3,
4571 uint8_t OldImm =
Imm;
4572 Imm = OldImm & 0x99;
4573 if (OldImm & 0x02)
Imm |= 0x04;
4574 if (OldImm & 0x04)
Imm |= 0x02;
4575 if (OldImm & 0x20)
Imm |= 0x40;
4576 if (OldImm & 0x40)
Imm |= 0x20;
4581 SDValue TImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
4587 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
4591 auto *MemIntr = cast<MemIntrinsicSDNode>(
C);
4592 unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits();
4593 assert((EltSize == 32 || EltSize == 64) &&
"Unexpected broadcast size!");
4595 bool UseD = EltSize == 32;
4597 Opc = UseD ? X86::VPTERNLOGDZ128rmbi : X86::VPTERNLOGQZ128rmbi;
4599 Opc = UseD ? X86::VPTERNLOGDZ256rmbi : X86::VPTERNLOGQZ256rmbi;
4601 Opc = UseD ? X86::VPTERNLOGDZrmbi : X86::VPTERNLOGQZrmbi;
4607 Opc = UseD ? X86::VPTERNLOGDZ128rmi : X86::VPTERNLOGQZ128rmi;
4609 Opc = UseD ? X86::VPTERNLOGDZ256rmi : X86::VPTERNLOGQZ256rmi;
4611 Opc = UseD ? X86::VPTERNLOGDZrmi : X86::VPTERNLOGQZrmi;
4616 SDValue Ops[] = {
A,
B, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, TImm,
C.getOperand(0)};
4617 MNode = CurDAG->getMachineNode(Opc,
DL, VTs, Ops);
4620 ReplaceUses(
C.getValue(1),
SDValue(MNode, 1));
4622 CurDAG->setNodeMemRefs(MNode, {cast<MemSDNode>(
C)->getMemOperand()});
4627 Opc = UseD ? X86::VPTERNLOGDZ128rri : X86::VPTERNLOGQZ128rri;
4629 Opc = UseD ? X86::VPTERNLOGDZ256rri : X86::VPTERNLOGQZ256rri;
4631 Opc = UseD ? X86::VPTERNLOGDZrri : X86::VPTERNLOGQZrri;
4635 MNode = CurDAG->getMachineNode(Opc,
DL, NVT, {
A,
B,
C, TImm});
4639 CurDAG->RemoveDeadNode(Root);
4645bool X86DAGToDAGISel::tryVPTERNLOG(
SDNode *
N) {
4646 MVT NVT =
N->getSimpleValueType(0);
4649 if (!NVT.
isVector() || !Subtarget->hasAVX512() ||
4660 auto getFoldableLogicOp = [](
SDValue Op) {
4663 Op =
Op.getOperand(0);
4665 if (!
Op.hasOneUse())
4668 unsigned Opc =
Op.getOpcode();
4677 if ((FoldableOp = getFoldableLogicOp(N1))) {
4679 }
else if ((FoldableOp = getFoldableLogicOp(N0))) {
4692 uint8_t TernlogMagicA = 0xf0;
4693 uint8_t TernlogMagicB = 0xcc;
4694 uint8_t TernlogMagicC = 0xaa;
4703 Parent =
Op.getNode();
4704 Op =
Op.getOperand(0);
4708 PeekThroughNot(
A, ParentA, TernlogMagicA);
4709 PeekThroughNot(
B, ParentB, TernlogMagicB);
4710 PeekThroughNot(
C, ParentC, TernlogMagicC);
4715 case ISD::AND:
Imm = TernlogMagicB & TernlogMagicC;
break;
4716 case ISD::OR:
Imm = TernlogMagicB | TernlogMagicC;
break;
4717 case ISD::XOR:
Imm = TernlogMagicB ^ TernlogMagicC;
break;
4721 switch (
N->getOpcode()) {
4725 Imm &= ~TernlogMagicA;
4727 Imm = ~(
Imm) & TernlogMagicA;
4734 return matchVPTERNLOG(
N, ParentA, ParentB, ParentC,
A,
B,
C, Imm);
4744bool X86DAGToDAGISel::shrinkAndImmediate(
SDNode *
And) {
4747 MVT VT =
And->getSimpleValueType(0);
4748 if (VT != MVT::i32 && VT != MVT::i64)
4751 auto *And1C = dyn_cast<ConstantSDNode>(
And->getOperand(1));
4760 APInt MaskVal = And1C->getAPIntValue();
4762 if (!MaskLZ || (VT == MVT::i64 && MaskLZ == 32))
4766 if (VT == MVT::i64 && MaskLZ >= 32) {
4768 MaskVal = MaskVal.
trunc(32);
4773 APInt NegMaskVal = MaskVal | HighZeros;
4782 if (VT == MVT::i64 && MaskVal.
getBitWidth() < 64) {
4783 NegMaskVal = NegMaskVal.
zext(64);
4784 HighZeros = HighZeros.
zext(64);
4789 if (!CurDAG->MaskedValueIsZero(And0, HighZeros))
4809 bool FoldedBCast,
bool Masked) {
4810#define VPTESTM_CASE(VT, SUFFIX) \
4813 return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
4814 return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
4817#define VPTESTM_BROADCAST_CASES(SUFFIX) \
4818default: llvm_unreachable("Unexpected VT!"); \
4819VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
4820VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
4821VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
4822VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
4823VPTESTM_CASE(v16i32, DZ##SUFFIX) \
4824VPTESTM_CASE(v8i64, QZ##SUFFIX)
4826#define VPTESTM_FULL_CASES(SUFFIX) \
4827VPTESTM_BROADCAST_CASES(SUFFIX) \
4828VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
4829VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
4830VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
4831VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
4832VPTESTM_CASE(v64i8, BZ##SUFFIX) \
4833VPTESTM_CASE(v32i16, WZ##SUFFIX)
4851#undef VPTESTM_FULL_CASES
4852#undef VPTESTM_BROADCAST_CASES
4858bool X86DAGToDAGISel::tryVPTESTM(
SDNode *Root,
SDValue Setcc,
4860 assert(Subtarget->hasAVX512() &&
"Expected AVX512!");
4910 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4915 if (CmpSVT != MVT::i32 && CmpSVT != MVT::i64)
4921 L =
L.getOperand(0);
4927 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4928 if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.
getSizeInBits())
4931 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4935 bool CanFoldLoads = Src0 != Src1;
4937 bool FoldedLoad =
false;
4938 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4940 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src1, Tmp0, Tmp1, Tmp2,
4944 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src0, Tmp0, Tmp1,
4953 bool IsMasked = InMask.
getNode() !=
nullptr;
4966 SDValue ImplDef =
SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl,
4968 Src0 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src0);
4971 Src1 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src1);
4976 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4977 InMask =
SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4978 dl, MaskVT, InMask, RC), 0);
4983 unsigned Opc =
getVPTESTMOpc(CmpVT, IsTestN, FoldedLoad, FoldedBCast,
4988 SDVTList VTs = CurDAG->getVTList(MaskVT, MVT::Other);
4991 SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4993 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
4995 SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4997 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5003 CurDAG->setNodeMemRefs(CNode, {cast<MemSDNode>(Src1)->getMemOperand()});
5006 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, InMask, Src0, Src1);
5008 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, Src0, Src1);
5014 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
5015 CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
5016 dl, ResVT,
SDValue(CNode, 0), RC);
5020 CurDAG->RemoveDeadNode(Root);
5026bool X86DAGToDAGISel::tryMatchBitSelect(
SDNode *
N) {
5029 MVT NVT =
N->getSimpleValueType(0);
5032 if (!NVT.
isVector() || !Subtarget->hasAVX512())
5066 SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8);
5075 MVT NVT =
Node->getSimpleValueType(0);
5076 unsigned Opcode =
Node->getOpcode();
5079 if (
Node->isMachineOpcode()) {
5081 Node->setNodeId(-1);
5088 unsigned IntNo =
Node->getConstantOperandVal(1);
5091 case Intrinsic::x86_encodekey128:
5092 case Intrinsic::x86_encodekey256: {
5093 if (!Subtarget->hasKL())
5099 case Intrinsic::x86_encodekey128:
5100 Opcode = X86::ENCODEKEY128;
5102 case Intrinsic::x86_encodekey256:
5103 Opcode = X86::ENCODEKEY256;
5108 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(3),
5110 if (Opcode == X86::ENCODEKEY256)
5111 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(4),
5115 Opcode, dl,
Node->getVTList(),
5116 {Node->getOperand(2), Chain, Chain.getValue(1)});
5117 ReplaceNode(
Node, Res);
5120 case Intrinsic::x86_tileloadd64_internal:
5121 case Intrinsic::x86_tileloaddt164_internal: {
5122 if (!Subtarget->hasAMXTILE())
5127 unsigned Opc = IntNo == Intrinsic::x86_tileloadd64_internal
5129 : X86::PTILELOADDT1V;
5132 SDValue Scale = getI8Imm(1, dl);
5134 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5135 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5139 Node->getOperand(3),
5146 CNode = CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops);
5147 ReplaceNode(
Node, CNode);
5154 unsigned IntNo =
Node->getConstantOperandVal(1);
5157 case Intrinsic::x86_sse3_monitor:
5158 case Intrinsic::x86_monitorx:
5159 case Intrinsic::x86_clzero: {
5160 bool Use64BitPtr =
Node->getOperand(2).getValueType() == MVT::i64;
5165 case Intrinsic::x86_sse3_monitor:
5166 if (!Subtarget->hasSSE3())
5168 Opc = Use64BitPtr ? X86::MONITOR64rrr : X86::MONITOR32rrr;
5170 case Intrinsic::x86_monitorx:
5171 if (!Subtarget->hasMWAITX())
5173 Opc = Use64BitPtr ? X86::MONITORX64rrr : X86::MONITORX32rrr;
5175 case Intrinsic::x86_clzero:
5176 if (!Subtarget->hasCLZERO())
5178 Opc = Use64BitPtr ? X86::CLZERO64r : X86::CLZERO32r;
5183 unsigned PtrReg = Use64BitPtr ? X86::RAX : X86::EAX;
5184 SDValue Chain = CurDAG->getCopyToReg(
Node->getOperand(0), dl, PtrReg,
5188 if (IntNo == Intrinsic::x86_sse3_monitor ||
5189 IntNo == Intrinsic::x86_monitorx) {
5191 Chain = CurDAG->getCopyToReg(Chain, dl, X86::ECX,
Node->getOperand(3),
5194 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EDX,
Node->getOperand(4),
5199 MachineSDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
5201 ReplaceNode(
Node, CNode);
5207 case Intrinsic::x86_tilestored64_internal: {
5211 unsigned Opc = X86::PTILESTOREDV;
5214 SDValue Scale = getI8Imm(1, dl);
5216 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5217 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5221 Node->getOperand(3),
5227 Node->getOperand(6),
5229 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5230 ReplaceNode(
Node, CNode);
5233 case Intrinsic::x86_tileloadd64:
5234 case Intrinsic::x86_tileloaddt164:
5235 case Intrinsic::x86_tilestored64: {
5236 if (!Subtarget->hasAMXTILE())
5244 case Intrinsic::x86_tileloadd64: Opc = X86::PTILELOADD;
break;
5245 case Intrinsic::x86_tileloaddt164: Opc = X86::PTILELOADDT1;
break;
5246 case Intrinsic::x86_tilestored64: Opc = X86::PTILESTORED;
break;
5249 unsigned TIndex =
Node->getConstantOperandVal(2);
5250 SDValue TReg = getI8Imm(TIndex, dl);
5252 SDValue Scale = getI8Imm(1, dl);
5254 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5255 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5258 if (Opc == X86::PTILESTORED) {
5260 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5263 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5265 ReplaceNode(
Node, CNode);
5273 if (Subtarget->isTargetNaCl())
5277 if (Subtarget->isTarget64BitILP32()) {
5282 assert(
Target.getValueType() == MVT::i32 &&
"Unexpected VT!");
5283 SDValue ZextTarget = CurDAG->getZExtOrTrunc(
Target, dl, MVT::i64);
5284 SDValue Brind = CurDAG->getNode(Opcode, dl, MVT::Other,
5285 Node->getOperand(0), ZextTarget);
5287 SelectCode(ZextTarget.
getNode());
5294 ReplaceNode(
Node, getGlobalBaseReg());
5302 CurDAG->RemoveDeadNode(
Node);
5308 if (matchBitExtract(
Node))
5313 if (tryShiftAmountMod(
Node))
5318 uint8_t
Imm =
Node->getConstantOperandVal(3);
5320 Node->getOperand(1),
Node->getOperand(2), Imm))
5326 if (tryVPTERNLOG(
Node))
5336 tryVPTESTM(
Node, N0, N1))
5339 tryVPTESTM(
Node, N1, N0))
5345 CurDAG->RemoveDeadNode(
Node);
5348 if (matchBitExtract(
Node))
5356 if (tryShrinkShlLogicImm(
Node))
5360 if (tryVPTERNLOG(
Node))
5375 if (!CurDAG->shouldOptForSize())
5379 if (NVT != MVT::i8 && NVT != MVT::i16 && NVT != MVT::i32 && NVT != MVT::i64)
5385 auto *Cst = dyn_cast<ConstantSDNode>(N1);
5389 int64_t Val = Cst->getSExtValue();
5393 if (!isInt<8>(Val) && !isInt<32>(Val))
5397 if (Opcode ==
ISD::ADD && (Val == 1 || Val == -1))
5401 if (!shouldAvoidImmediateInstFormsForSize(N1.
getNode()))
5405 unsigned ROpc, MOpc;
5514 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5515 if (tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5517 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5518 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5522 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N0)->getMemOperand()});
5524 CurDAG->RemoveDeadNode(
Node);
5529 CurDAG->SelectNodeTo(
Node, ROpc, NVT, MVT::i32, N0, N1);
5542 unsigned LoReg, ROpc, MOpc;
5547 ROpc = Opcode ==
X86ISD::SMUL ? X86::IMUL8r : X86::MUL8r;
5548 MOpc = Opcode ==
X86ISD::SMUL ? X86::IMUL8m : X86::MUL8m;
5567 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5568 bool FoldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5571 FoldedLoad = tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5576 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5585 VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5587 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32, MVT::Other);
5591 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5596 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5602 VTs = CurDAG->getVTList(NVT, MVT::i32);
5604 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32);
5606 CNode = CurDAG->getMachineNode(ROpc, dl, VTs, {N1, InGlue});
5611 CurDAG->RemoveDeadNode(
Node);
5621 unsigned LoReg, HiReg;
5623 bool UseMULX = !IsSigned && Subtarget->hasBMI2();
5628 Opc = UseMULXHi ? X86::MULX32Hrr
5630 : IsSigned ?
X86::IMUL32r
5632 MOpc = UseMULXHi ? X86::MULX32Hrm
5634 : IsSigned ?
X86::IMUL32m
5636 LoReg = UseMULX ? X86::EDX : X86::EAX;
5640 Opc = UseMULXHi ? X86::MULX64Hrr
5642 : IsSigned ?
X86::IMUL64r
5644 MOpc = UseMULXHi ? X86::MULX64Hrm
5646 : IsSigned ?
X86::IMUL64m
5648 LoReg = UseMULX ? X86::RDX : X86::RAX;
5653 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5654 bool foldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5657 foldedLoad = tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5662 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5671 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
5672 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5675 }
else if (UseMULX) {
5676 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other);
5677 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5682 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5683 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5689 ReplaceUses(N1.
getValue(1), Chain);
5691 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5693 SDValue Ops[] = { N1, InGlue };
5695 SDVTList VTs = CurDAG->getVTList(NVT);
5696 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5698 }
else if (UseMULX) {
5699 SDVTList VTs = CurDAG->getVTList(NVT, NVT);
5700 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5704 SDVTList VTs = CurDAG->getVTList(MVT::Glue);
5705 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5713 assert(LoReg &&
"Register for low half is not defined!");
5714 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg,
5725 assert(HiReg &&
"Register for high half is not defined!");
5726 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg,
5735 CurDAG->RemoveDeadNode(
Node);
5744 unsigned ROpc, MOpc;
5749 case MVT::i8: ROpc = X86::DIV8r; MOpc = X86::DIV8m;
break;
5750 case MVT::i16: ROpc = X86::DIV16r; MOpc = X86::DIV16m;
break;
5751 case MVT::i32: ROpc = X86::DIV32r; MOpc = X86::DIV32m;
break;
5752 case MVT::i64: ROpc = X86::DIV64r; MOpc = X86::DIV64m;
break;
5757 case MVT::i8: ROpc = X86::IDIV8r; MOpc = X86::IDIV8m;
break;
5758 case MVT::i16: ROpc = X86::IDIV16r; MOpc = X86::IDIV16m;
break;
5759 case MVT::i32: ROpc = X86::IDIV32r; MOpc = X86::IDIV32m;
break;
5760 case MVT::i64: ROpc = X86::IDIV64r; MOpc = X86::IDIV64m;
break;
5764 unsigned LoReg, HiReg, ClrReg;
5765 unsigned SExtOpcode;
5769 LoReg = X86::AL; ClrReg = HiReg = X86::AH;
5773 LoReg = X86::AX; HiReg = X86::DX;
5775 SExtOpcode = X86::CWD;
5778 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX;
5779 SExtOpcode = X86::CDQ;
5782 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX;
5783 SExtOpcode = X86::CQO;
5787 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5788 bool foldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5789 bool signBitIsZero = CurDAG->SignBitIsZero(N0);
5792 if (NVT == MVT::i8) {
5795 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain;
5797 if (tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5799 unsigned Opc = (
isSigned && !signBitIsZero) ? X86::MOVSX16rm8
5801 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, MVT::Other, Ops);
5803 ReplaceUses(N0.
getValue(1), Chain);
5805 CurDAG->setNodeMemRefs(Move, {cast<LoadSDNode>(N0)->getMemOperand()});
5807 unsigned Opc = (
isSigned && !signBitIsZero) ? X86::MOVSX16rr8
5809 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, N0);
5810 Chain = CurDAG->getEntryNode();
5812 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX,
SDValue(Move, 0),
5817 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
5818 LoReg, N0,
SDValue()).getValue(1);
5822 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InGlue),0);
5825 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
5827 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
5831 SDValue(CurDAG->getMachineNode(
5832 TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode,
5833 CurDAG->getTargetConstant(X86::sub_16bit, dl,
5841 SDValue(CurDAG->getMachineNode(
5842 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
5843 CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode,
5844 CurDAG->getTargetConstant(X86::sub_32bit, dl,
5852 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg,
5853 ClrNode, InGlue).getValue(1);
5861 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops);
5866 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5869 SDValue(CurDAG->getMachineNode(ROpc, dl, MVT::Glue, N1, InGlue), 0);
5879 if (HiReg == X86::AH && !
SDValue(
Node, 1).use_empty()) {
5880 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8);
5881 unsigned AHExtOpcode =
5882 isSigned ? X86::MOVSX32rr8_NOREX : X86::MOVZX32rr8_NOREX;
5884 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32,
5885 MVT::Glue, AHCopy, InGlue);
5890 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result);
5898 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5899 LoReg, NVT, InGlue);
5900 InGlue =
Result.getValue(2);
5907 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5908 HiReg, NVT, InGlue);
5909 InGlue =
Result.getValue(2);
5914 CurDAG->RemoveDeadNode(
Node);
5923 SDValue N0 =
Node->getOperand(IsStrictCmp ? 1 : 0);
5924 SDValue N1 =
Node->getOperand(IsStrictCmp ? 2 : 1);
5930 if (Subtarget->canUseCMOV())
5939 Opc = IsSignaling ? X86::COM_Fpr32 : X86::UCOM_Fpr32;
5942 Opc = IsSignaling ? X86::COM_Fpr64 : X86::UCOM_Fpr64;
5945 Opc = IsSignaling ? X86::COM_Fpr80 : X86::UCOM_Fpr80;
5950 IsStrictCmp ?
Node->getOperand(0) : CurDAG->getEntryNode();
5953 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5954 Chain =
SDValue(CurDAG->getMachineNode(Opc, dl, VTs, {N0, N1, Chain}), 0);
5957 Glue =
SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N0, N1), 0);
5962 SDValue(CurDAG->getMachineNode(X86::FNSTSW16r, dl, MVT::i16, Glue), 0);
5966 CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, MVT::i8, FNSTSW);
5970 assert(Subtarget->canUseLAHFSAHF() &&
5971 "Target doesn't support SAHF or FCOMI?");
5972 SDValue AH = CurDAG->getCopyToReg(Chain, dl, X86::AH, Extract,
SDValue());
5975 CurDAG->getMachineNode(X86::SAHF, dl, MVT::i32, AH.
getValue(1)), 0);
5981 CurDAG->RemoveDeadNode(
Node);
6002 unsigned TestOpc = CmpVT == MVT::i64 ? X86::TEST64rr
6005 NewNode = CurDAG->getMachineNode(TestOpc, dl, MVT::i32, BEXTR, BEXTR);
6007 CurDAG->RemoveDeadNode(
Node);
6021 auto *MaskC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
6039 unsigned TestOpcode;
6047 if (LeadingZeros == 0 && SavesBytes) {
6052 ShiftAmt = TrailingZeros;
6054 TestOpcode = X86::TEST64rr;
6055 }
else if (TrailingZeros == 0 && SavesBytes) {
6060 ShiftAmt = LeadingZeros;
6062 TestOpcode = X86::TEST64rr;
6063 }
else if (MaskC->hasOneUse() && !isInt<32>(Mask)) {
6066 unsigned PopCount = 64 - LeadingZeros - TrailingZeros;
6067 if (PopCount == 8) {
6069 ShiftAmt = TrailingZeros;
6070 SubRegIdx = X86::sub_8bit;
6072 TestOpcode = X86::TEST8rr;
6073 }
else if (PopCount == 16) {
6075 ShiftAmt = TrailingZeros;
6076 SubRegIdx = X86::sub_16bit;
6077 SubRegVT = MVT::i16;
6078 TestOpcode = X86::TEST16rr;
6079 }
else if (PopCount == 32) {
6081 ShiftAmt = TrailingZeros;
6082 SubRegIdx = X86::sub_32bit;
6083 SubRegVT = MVT::i32;
6084 TestOpcode = X86::TEST32rr;
6088 SDValue ShiftC = CurDAG->getTargetConstant(ShiftAmt, dl, MVT::i64);
6090 CurDAG->getMachineNode(ShiftOpcode, dl, MVT::i64, MVT::i32,
6093 if (SubRegIdx != 0) {
6095 CurDAG->getTargetExtractSubreg(SubRegIdx, dl, SubRegVT, Shift);
6098 CurDAG->getMachineNode(TestOpcode, dl, MVT::i32, Shift, Shift);
6106 unsigned ROpc, MOpc;
6113 if (isUInt<8>(Mask) &&
6114 (!(Mask & 0x80) || CmpVT == MVT::i8 ||
6118 SubRegOp = X86::sub_8bit;
6119 ROpc = X86::TEST8ri;
6120 MOpc = X86::TEST8mi;
6121 }
else if (OptForMinSize && isUInt<16>(Mask) &&
6122 (!(Mask & 0x8000) || CmpVT == MVT::i16 ||
6129 SubRegOp = X86::sub_16bit;
6130 ROpc = X86::TEST16ri;
6131 MOpc = X86::TEST16mi;
6132 }
else if (isUInt<32>(Mask) && N0.
getValueType() != MVT::i16 &&
6133 ((!(Mask & 0x80000000) &&
6136 (CmpVT != MVT::i16 || !(Mask & 0x8000))) ||
6137 CmpVT == MVT::i32 ||
6145 SubRegOp = X86::sub_32bit;
6146 ROpc = X86::TEST32ri;
6147 MOpc = X86::TEST32mi;
6153 SDValue Imm = CurDAG->getTargetConstant(Mask, dl, VT);
6158 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
6159 if (tryFoldLoad(
Node, N0.
getNode(), Reg, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
6161 if (!LoadN->isSimple()) {
6163 if ((MOpc == X86::TEST8mi && NumVolBits != 8) ||
6164 (MOpc == X86::TEST16mi && NumVolBits != 16) ||
6165 (MOpc == X86::TEST32mi && NumVolBits != 32))
6169 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
6170 Reg.getOperand(0) };
6171 NewNode = CurDAG->getMachineNode(MOpc, dl, MVT::i32, MVT::Other, Ops);
6173 ReplaceUses(
Reg.getValue(1),
SDValue(NewNode, 1));
6175 CurDAG->setNodeMemRefs(NewNode,
6176 {cast<LoadSDNode>(Reg)->getMemOperand()});
6180 Reg = CurDAG->getTargetExtractSubreg(SubRegOp, dl, VT, Reg);
6182 NewNode = CurDAG->getMachineNode(ROpc, dl, MVT::i32, Reg, Imm);
6185 ReplaceNode(
Node, NewNode);
6191 if (!Subtarget->hasSSE42())
6197 bool MayFoldLoad = !NeedIndex || !NeedMask;
6202 Subtarget->hasAVX() ? X86::VPCMPISTRMrri : X86::PCMPISTRMrri;
6204 Subtarget->hasAVX() ? X86::VPCMPISTRMrmi : X86::PCMPISTRMrmi;
6205 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8,
Node);
6208 if (NeedIndex || !NeedMask) {
6210 Subtarget->hasAVX() ? X86::VPCMPISTRIrri : X86::PCMPISTRIrri;
6212 Subtarget->hasAVX() ? X86::VPCMPISTRIrmi : X86::PCMPISTRIrmi;
6213 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32,
Node);
6219 CurDAG->RemoveDeadNode(
Node);
6223 if (!Subtarget->hasSSE42())
6227 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX,
6228 Node->getOperand(1),
6230 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EDX,
6231 Node->getOperand(3), InGlue).getValue(1);
6236 bool MayFoldLoad = !NeedIndex || !NeedMask;
6241 Subtarget->hasAVX() ? X86::VPCMPESTRMrri : X86::PCMPESTRMrri;
6243 Subtarget->hasAVX() ? X86::VPCMPESTRMrmi : X86::PCMPESTRMrmi;
6245 emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8,
Node, InGlue);
6248 if (NeedIndex || !NeedMask) {
6250 Subtarget->hasAVX() ? X86::VPCMPESTRIrri : X86::PCMPESTRIrri;
6252 Subtarget->hasAVX() ? X86::VPCMPESTRIrmi : X86::PCMPESTRIrmi;
6253 CNode = emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32,
Node, InGlue);
6258 CurDAG->RemoveDeadNode(
Node);
6270 if (foldLoadStoreIntoMemOperand(
Node))
6275 MVT VT =
Node->getSimpleValueType(0);
6277 if (Subtarget->hasSBBDepBreaking()) {
6282 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
6287 unsigned Opc = VT == MVT::i64 ? X86::SETB_C64r : X86::SETB_C32r;
6288 MVT SetVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
6290 CurDAG->getMachineNode(Opc, dl, SetVT, EFLAGS, EFLAGS.
getValue(1)),
6299 if (VT == MVT::i8 || VT == MVT::i16) {
6300 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6301 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6305 CurDAG->RemoveDeadNode(
Node);
6319 MVT VT =
Node->getSimpleValueType(0);
6320 if (VT == MVT::i8 || VT == MVT::i16) {
6321 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6322 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6327 CurDAG->RemoveDeadNode(
Node);
6333 auto *Mgt = cast<X86MaskedGatherSDNode>(
Node);
6334 SDValue IndexOp = Mgt->getIndex();
6337 MVT ValueVT =
Node->getSimpleValueType(0);
6338 MVT MaskVT =
Mask.getSimpleValueType();
6355 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6356 Opc = IsFP ? X86::VGATHERDPSZ128rm : X86::VPGATHERDDZ128rm;
6357 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6358 Opc = IsFP ? X86::VGATHERDPSZ256rm : X86::VPGATHERDDZ256rm;
6359 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6360 Opc = IsFP ? X86::VGATHERDPSZrm : X86::VPGATHERDDZrm;
6361 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6362 Opc = IsFP ? X86::VGATHERDPDZ128rm : X86::VPGATHERDQZ128rm;
6363 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6364 Opc = IsFP ? X86::VGATHERDPDZ256rm : X86::VPGATHERDQZ256rm;
6365 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6366 Opc = IsFP ? X86::VGATHERDPDZrm : X86::VPGATHERDQZrm;
6367 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6368 Opc = IsFP ? X86::VGATHERQPSZ128rm : X86::VPGATHERQDZ128rm;
6369 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6370 Opc = IsFP ? X86::VGATHERQPSZ256rm : X86::VPGATHERQDZ256rm;
6371 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6372 Opc = IsFP ? X86::VGATHERQPSZrm : X86::VPGATHERQDZrm;
6373 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6374 Opc = IsFP ? X86::VGATHERQPDZ128rm : X86::VPGATHERQQZ128rm;
6375 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6376 Opc = IsFP ? X86::VGATHERQPDZ256rm : X86::VPGATHERQQZ256rm;
6377 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6378 Opc = IsFP ? X86::VGATHERQPDZrm : X86::VPGATHERQQZrm;
6380 assert(
EVT(MaskVT) ==
EVT(ValueVT).changeVectorElementTypeToInteger() &&
6381 "Unexpected mask VT!");
6382 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6383 Opc = IsFP ? X86::VGATHERDPSrm : X86::VPGATHERDDrm;
6384 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6385 Opc = IsFP ? X86::VGATHERDPSYrm : X86::VPGATHERDDYrm;
6386 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6387 Opc = IsFP ? X86::VGATHERDPDrm : X86::VPGATHERDQrm;
6388 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6389 Opc = IsFP ? X86::VGATHERDPDYrm : X86::VPGATHERDQYrm;
6390 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6391 Opc = IsFP ? X86::VGATHERQPSrm : X86::VPGATHERQDrm;
6392 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6393 Opc = IsFP ? X86::VGATHERQPSYrm : X86::VPGATHERQDYrm;
6394 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6395 Opc = IsFP ? X86::VGATHERQPDrm : X86::VPGATHERQQrm;
6396 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6397 Opc = IsFP ? X86::VGATHERQPDYrm : X86::VPGATHERQQYrm;
6404 if (!selectVectorAddr(Mgt, Mgt->getBasePtr(), IndexOp, Mgt->getScale(),
6408 SDValue PassThru = Mgt->getPassThru();
6409 SDValue Chain = Mgt->getChain();
6411 SDVTList VTs = CurDAG->getVTList(ValueVT, MaskVT, MVT::Other);
6416 Index, Disp, Segment, Chain};
6417 NewNode = CurDAG->getMachineNode(Opc,
SDLoc(dl), VTs, Ops);
6420 Disp, Segment,
Mask, Chain};
6421 NewNode = CurDAG->getMachineNode(Opc,
SDLoc(dl), VTs, Ops);
6423 CurDAG->setNodeMemRefs(NewNode, {Mgt->getMemOperand()});
6426 CurDAG->RemoveDeadNode(
Node);
6430 auto *Sc = cast<X86MaskedScatterSDNode>(
Node);
6432 SDValue IndexOp = Sc->getIndex();
6434 MVT ValueVT =
Value.getSimpleValueType();
6449 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6450 Opc = IsFP ? X86::VSCATTERDPSZ128mr : X86::VPSCATTERDDZ128mr;
6451 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6452 Opc = IsFP ? X86::VSCATTERDPSZ256mr : X86::VPSCATTERDDZ256mr;
6453 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6454 Opc = IsFP ? X86::VSCATTERDPSZmr : X86::VPSCATTERDDZmr;
6455 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6456 Opc = IsFP ? X86::VSCATTERDPDZ128mr : X86::VPSCATTERDQZ128mr;
6457 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6458 Opc = IsFP ? X86::VSCATTERDPDZ256mr : X86::VPSCATTERDQZ256mr;
6459 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6460 Opc = IsFP ? X86::VSCATTERDPDZmr : X86::VPSCATTERDQZmr;
6461 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6462 Opc = IsFP ? X86::VSCATTERQPSZ128mr : X86::VPSCATTERQDZ128mr;
6463 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6464 Opc = IsFP ? X86::VSCATTERQPSZ256mr : X86::VPSCATTERQDZ256mr;
6465 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6466 Opc = IsFP ? X86::VSCATTERQPSZmr : X86::VPSCATTERQDZmr;
6467 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6468 Opc = IsFP ? X86::VSCATTERQPDZ128mr : X86::VPSCATTERQQZ128mr;
6469 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6470 Opc = IsFP ? X86::VSCATTERQPDZ256mr : X86::VPSCATTERQQZ256mr;
6471 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6472 Opc = IsFP ? X86::VSCATTERQPDZmr : X86::VPSCATTERQQZmr;
6477 if (!selectVectorAddr(Sc, Sc->getBasePtr(), IndexOp, Sc->getScale(),
6482 SDValue Chain = Sc->getChain();
6484 SDVTList VTs = CurDAG->getVTList(
Mask.getValueType(), MVT::Other);
6488 CurDAG->setNodeMemRefs(NewNode, {Sc->getMemOperand()});
6490 CurDAG->RemoveDeadNode(
Node);
6496 cast<SrcValueSDNode>(
Node->getOperand(1))->getValue());
6498 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6500 TargetOpcode::PREALLOCATED_SETUP, dl, MVT::Other, CallIdValue, Chain);
6502 CurDAG->RemoveDeadNode(
Node);
6508 cast<SrcValueSDNode>(
Node->getOperand(1))->getValue());
6510 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6513 Ops[0] = CallIdValue;
6517 TargetOpcode::PREALLOCATED_ARG, dl,
6518 CurDAG->getVTList(TLI->
getPointerTy(CurDAG->getDataLayout()),
6523 CurDAG->RemoveDeadNode(
Node);
6530 if (!Subtarget->hasWIDEKL())
6534 switch (
Node->getOpcode()) {
6538 Opcode = X86::AESENCWIDE128KL;
6541 Opcode = X86::AESDECWIDE128KL;
6544 Opcode = X86::AESENCWIDE256KL;
6547 Opcode = X86::AESDECWIDE256KL;
6558 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(2),
6560 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(3),
6562 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM2,
Node->getOperand(4),
6564 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM3,
Node->getOperand(5),
6566 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM4,
Node->getOperand(6),
6568 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM5,
Node->getOperand(7),
6570 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM6,
Node->getOperand(8),
6572 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM7,
Node->getOperand(9),
6576 Opcode, dl,
Node->getVTList(),
6577 {Base, Scale, Index, Disp, Segment, Chain, Chain.getValue(1)});
6578 CurDAG->setNodeMemRefs(Res, cast<MemSDNode>(
Node)->getMemOperand());
6579 ReplaceNode(
Node, Res);
6587bool X86DAGToDAGISel::SelectInlineAsmMemoryOperand(
6589 std::vector<SDValue> &OutOps) {
6590 SDValue Op0, Op1, Op2, Op3, Op4;
6591 switch (ConstraintID) {
6594 case InlineAsm::ConstraintCode::o:
6595 case InlineAsm::ConstraintCode::v:
6596 case InlineAsm::ConstraintCode::m:
6597 case InlineAsm::ConstraintCode::X:
6598 case InlineAsm::ConstraintCode::p:
6599 if (!selectAddr(
nullptr,
Op, Op0, Op1, Op2, Op3, Op4))
6604 OutOps.push_back(Op0);
6605 OutOps.push_back(Op1);
6606 OutOps.push_back(Op2);
6607 OutOps.push_back(Op3);
6608 OutOps.push_back(Op4);
6614 std::make_unique<X86DAGToDAGISel>(
TM,
TM.getOptLevel())) {}
6620 return new X86DAGToDAGISelLegacy(
TM, OptLevel);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
amdgpu AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Module.h This file contains the declarations for the Module class.
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool isRIPRelative(const MCInst &MI, const MCInstrInfo &MCII)
Check if the instruction uses RIP relative addressing.
static bool isLegalMaskCompare(SDNode *N, const X86Subtarget *Subtarget)
static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, unsigned LoadOpNo, LoadSDNode *&LoadNode, SDValue &InputChain)
Check whether or not the chain ending in StoreNode is suitable for doing the {load; op; store} to mod...
#define GET_EGPR_IF_ENABLED(OPC)
static bool needBWI(MVT VT)
static unsigned getVPTESTMOpc(MVT TestVT, bool IsTestN, bool FoldedLoad, bool FoldedBCast, bool Masked)
static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM, const X86Subtarget &Subtarget)
static bool mayUseCarryFlag(X86::CondCode CC)
static cl::opt< bool > EnablePromoteAnyextLoad("x86-promote-anyext-load", cl::init(true), cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden)
cl::opt< bool > IndirectBranchTracking
static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, SDValue Call, SDValue OrigChain)
Replace the original chain operand of the call with load's chain operand and move load below the call...
static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N)
#define GET_ND_IF_ENABLED(OPC)
#define VPTESTM_BROADCAST_CASES(SUFFIX)
static cl::opt< bool > AndImmShrink("x86-and-imm-shrink", cl::init(true), cl::desc("Enable setting constant bits to reduce size of mask immediates"), cl::Hidden)
static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, X86ISelAddressMode &AM)
#define VPTESTM_FULL_CASES(SUFFIX)
static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq)
Return true if call address is a load and it can be moved below CALLSEQ_START and the chains leading ...
static bool isDispSafeForFrameIndex(int64_t Val)
static bool isEndbrImm64(uint64_t Imm)
#define GET_ND_IF_ENABLED(OPC)
DEMANGLE_DUMP_METHOD void dump() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
APInt trunc(unsigned width) const
Truncate to new width.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countl_zero() const
The APInt version of std::countl_zero.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
bool isOne() const
Determine if this is a value of 1.
unsigned countr_one() const
Count the number of trailing one bits.
The address of a basic block.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
std::optional< ConstantRange > getAbsoluteSymbolRange() const
If this is an absolute symbol reference, returns the range of the symbol, otherwise returns std::null...
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
Describe properties that are true of each instruction in the target description file.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
bool is128BitVector() const
Return true if this is a 128-bit vector type.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool is512BitVector() const
Return true if this is a 512-bit vector type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const Module * getModule() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isNonTemporal() const
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
static int getUninvalidatedNodeId(SDNode *N)
virtual void emitFunctionEntryCode()
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
virtual bool runOnMachineFunction(MachineFunction &mf)
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
static constexpr unsigned MaxRecursionDepth
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
static Type * getVoidTy(LLVMContext &C)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void dump() const
Support for debugging, callable in GDB: V->dump()
X86ISelDAGToDAGPass(X86TargetMachine &TM)
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
void setAMXProgModel(AMXProgModelEnum Model)
size_t getPreallocatedIdForCallSite(const Value *CS)
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BRIND
BRIND - Indirect branch.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
@ GlobalBaseReg
The result of the mflr at function entry, used for PIC code.
@ X86
Windows x64, Windows Itanium (IA-64)
Reg
All possible values of the reg field in the ModR/M byte.
@ EVEX
EVEX - Specifies that this instruction use EVEX form which provides syntax support up to 32 512-bit r...
@ VEX
VEX - encoding using 0xC4/0xC5.
@ XOP
XOP - Opcode prefix used by XOP instructions.
@ MO_NO_FLAG
MO_NO_FLAG - No flag for the operand.
@ FST
This instruction implements a truncating store from FP stack slots.
@ CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ CMP
X86 compare and logical compare instructions.
@ BLENDV
Dynamic (non-constant condition) vector blend where only the sign bits of the condition elements are ...
@ STRICT_FCMP
X86 strict FP compare instructions.
@ STRICT_CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ NT_BRIND
BRIND node with NoTrack prefix.
@ FSETCCM
X86 FP SETCC, similar to above, but with output as an i1 mask and and a version with SAE.
@ FXOR
Bitwise logical XOR of floating point values.
@ BRCOND
X86 conditional branches.
@ CALL
These operations represent an abstract X86 call instruction, which includes a bunch of information.
@ FANDN
Bitwise logical ANDNOT of floating point values.
@ GlobalBaseReg
On Darwin, this node represents the result of the popl at function entry, used for PIC code.
@ FLD
This instruction implements an extending load to FP stack slots.
@ TC_RETURN
Tail call return.
@ FOR
Bitwise logical OR of floating point values.
@ Wrapper
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress,...
@ ANDNP
Bitwise Logical AND NOT of Packed FP values.
@ FAND
Bitwise logical AND of floating point values.
@ CMOV
X86 conditional moves.
@ WrapperRIP
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
int getCondSrcNoFromDesc(const MCInstrDesc &MCID)
Return the source operand # for condition code by MCID.
bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into some other x86 instruction as a memory oper...
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement)
Returns true of the given offset can be fit into displacement field of the instruction.
bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs)
If Op is a constant whose elements are all the same constant or undefined, return true and return the...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
int popcount(T Value) noexcept
Count the number of set bits in a value.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
unsigned M1(unsigned Val)
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
FunctionPass * createX86ISelDag(X86TargetMachine &TM, CodeGenOptLevel OptLevel)
This pass converts a legalized DAG into a X86-specific DAG, ready for instruction scheduling.
CodeGenOptLevel
Code generation optimization level.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.