22#include "llvm/Config/llvm-config.h"
27#include "llvm/IR/IntrinsicsX86.h"
37#define DEBUG_TYPE "x86-isel"
38#define PASS_NAME "X86 DAG->DAG Instruction Selection"
40STATISTIC(NumLoadMoved,
"Number of loads moved below TokenFactor");
43 cl::desc(
"Enable setting constant bits to reduce size of mask immediates"),
47 "x86-promote-anyext-load",
cl::init(
true),
59 struct X86ISelAddressMode {
67 int Base_FrameIndex = 0;
76 const char *ES =
nullptr;
81 bool NegateIndex =
false;
83 X86ISelAddressMode() =
default;
85 bool hasSymbolicDisplacement()
const {
86 return GV !=
nullptr ||
CP !=
nullptr || ES !=
nullptr ||
87 MCSym !=
nullptr ||
JT != -1 || BlockAddr !=
nullptr;
90 bool hasBaseOrIndexReg()
const {
97 if (
BaseType != RegBase)
return false;
99 dyn_cast_or_null<RegisterSDNode>(Base_Reg.
getNode()))
100 return RegNode->getReg() == X86::RIP;
109#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
111 dbgs() <<
"X86ISelAddressMode " <<
this <<
'\n';
112 dbgs() <<
"Base_Reg ";
118 dbgs() <<
" Base.FrameIndex " << Base_FrameIndex <<
'\n';
119 dbgs() <<
" Scale " << Scale <<
'\n'
127 dbgs() <<
" Disp " << Disp <<
'\n'
149 dbgs() <<
" JT" <<
JT <<
" Align" << Alignment.
value() <<
'\n';
169 bool IndirectTlsSegRefs;
174 X86DAGToDAGISel() =
delete;
178 OptForMinSize(
false), IndirectTlsSegRefs(
false) {}
184 "indirect-tls-seg-refs");
189 "OptForMinSize implies OptForSize");
203#include "X86GenDAGISel.inc"
208 bool foldOffsetIntoAddress(
uint64_t Offset, X86ISelAddressMode &AM);
209 bool matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
210 bool AllowSegmentRegForX32 =
false);
211 bool matchWrapper(
SDValue N, X86ISelAddressMode &AM);
212 bool matchAddress(
SDValue N, X86ISelAddressMode &AM);
213 bool matchVectorAddress(
SDValue N, X86ISelAddressMode &AM);
214 bool matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
unsigned Depth);
217 bool matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
219 bool matchVectorAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
221 bool matchAddressBase(
SDValue N, X86ISelAddressMode &AM);
250 return tryFoldLoad(
P,
P,
N,
Base, Scale,
Index, Disp, Segment);
258 bool isProfitableToFormMaskedOp(
SDNode *
N)
const;
263 std::vector<SDValue> &OutOps)
override;
265 void emitSpecialCodeForMain();
267 inline void getAddressOperands(X86ISelAddressMode &AM,
const SDLoc &
DL,
271 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
272 Base = CurDAG->getTargetFrameIndex(
273 AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout()));
274 else if (AM.Base_Reg.getNode())
277 Base = CurDAG->getRegister(0, VT);
279 Scale = getI8Imm(AM.Scale,
DL);
281#define GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC)
283 if (AM.NegateIndex) {
291 if (AM.IndexReg.getNode())
294 Index = CurDAG->getRegister(0, VT);
299 Disp = CurDAG->getTargetGlobalAddress(AM.GV,
SDLoc(),
303 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment,
304 AM.Disp, AM.SymbolFlags);
306 assert(!AM.Disp &&
"Non-zero displacement is ignored with ES.");
307 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
308 }
else if (AM.MCSym) {
309 assert(!AM.Disp &&
"Non-zero displacement is ignored with MCSym.");
310 assert(AM.SymbolFlags == 0 &&
"oo");
311 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32);
312 }
else if (AM.JT != -1) {
313 assert(!AM.Disp &&
"Non-zero displacement is ignored with JT.");
314 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
315 }
else if (AM.BlockAddr)
316 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
319 Disp = CurDAG->getTargetConstant(AM.Disp,
DL, MVT::i32);
321 if (AM.Segment.getNode())
322 Segment = AM.Segment;
324 Segment = CurDAG->getRegister(0, MVT::i16);
333 bool shouldAvoidImmediateInstFormsForSize(
SDNode *
N)
const {
339 if (!CurDAG->shouldOptForSize())
349 if (
User->isMachineOpcode()) {
372 auto *
C = dyn_cast<ConstantSDNode>(
N);
373 if (
C && isInt<8>(
C->getSExtValue()))
393 (RegNode = dyn_cast_or_null<RegisterSDNode>(
395 if ((RegNode->
getReg() == X86::ESP) ||
396 (RegNode->
getReg() == X86::RSP))
405 return (UseCount > 1);
410 return CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
415 return CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
420 return CurDAG->getTargetConstant(Imm,
DL, MVT::i64);
425 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
427 MVT VecVT =
N->getOperand(0).getSimpleValueType();
433 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
435 MVT VecVT =
N->getSimpleValueType(0);
439 SDValue getPermuteVINSERTCommutedImmediate(
SDNode *
N,
unsigned VecWidth,
441 assert(VecWidth == 128 &&
"Unexpected vector width");
443 MVT VecVT =
N->getSimpleValueType(0);
445 assert((InsertIdx == 0 || InsertIdx == 1) &&
"Bad insertf128 index");
448 return getI8Imm(InsertIdx ? 0x02 : 0x30,
DL);
453 MVT VT =
N->getSimpleValueType(0);
456 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
458 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
459 if (VT == MVT::i64) {
461 CurDAG->getMachineNode(
462 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
463 CurDAG->getTargetConstant(0, dl, MVT::i64), Zero,
464 CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)),
469 unsigned Opcode =
N->getOpcode();
471 "Unexpected opcode for SBB materialization");
472 unsigned FlagOpIndex = Opcode ==
X86ISD::SBB ? 2 : 1;
474 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
475 N->getOperand(FlagOpIndex),
SDValue());
479 unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr;
480 MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
481 VTs = CurDAG->getVTList(SBBVT, MVT::i32);
483 CurDAG->getMachineNode(Opc, dl, VTs,
484 {Zero, Zero, EFLAGS, EFLAGS.getValue(1)}),
490 bool isUnneededShiftMask(
SDNode *
N,
unsigned Width)
const {
492 const APInt &Val =
N->getConstantOperandAPInt(1);
497 APInt Mask = Val | CurDAG->computeKnownBits(
N->getOperand(0)).Zero;
498 return Mask.countr_one() >= Width;
504 SDNode *getGlobalBaseReg();
515 return Subtarget->getInstrInfo();
528 bool isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const;
532 if (!
N->isNonTemporal())
535 unsigned StoreSize =
N->getMemoryVT().getStoreSize();
537 if (
N->getAlign().value() < StoreSize)
546 return Subtarget->hasSSE41();
548 return Subtarget->hasAVX2();
550 return Subtarget->hasAVX512();
554 bool foldLoadStoreIntoMemOperand(
SDNode *
Node);
557 bool shrinkAndImmediate(
SDNode *
N);
558 bool isMaskZeroExtended(
SDNode *
N)
const;
559 bool tryShiftAmountMod(
SDNode *
N);
560 bool tryShrinkShlLogicImm(
SDNode *
N);
566 bool tryMatchBitSelect(
SDNode *
N);
568 MachineSDNode *emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
570 MachineSDNode *emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
574 bool tryOptimizeRem8Extend(
SDNode *
N);
576 bool onlyUsesZeroFlag(
SDValue Flags)
const;
577 bool hasNoSignFlagUses(
SDValue Flags)
const;
578 bool hasNoCarryFlagUses(
SDValue Flags)
const;
582char X86DAGToDAGISel::ID = 0;
589 unsigned Opcode =
N->getOpcode();
596 EVT OpVT =
N->getOperand(0).getValueType();
600 OpVT =
N->getOperand(1).getValueType();
602 return Subtarget->hasVLX();
616bool X86DAGToDAGISel::isMaskZeroExtended(
SDNode *
N)
const {
629 if (OptLevel == CodeGenOptLevel::None)
639 if (useNonTemporalLoad(cast<LoadSDNode>(
N)))
644 switch (
U->getOpcode()) {
670 if (
auto *Imm = dyn_cast<ConstantSDNode>(Op1)) {
671 if (
Imm->getAPIntValue().isSignedIntN(8))
680 Imm->getAPIntValue().getBitWidth() == 64 &&
681 Imm->getAPIntValue().isIntN(32))
688 (
Imm->getAPIntValue() == UINT8_MAX ||
689 Imm->getAPIntValue() == UINT16_MAX ||
690 Imm->getAPIntValue() == UINT32_MAX))
696 (-
Imm->getAPIntValue()).isSignedIntN(8))
700 (-
Imm->getAPIntValue()).isSignedIntN(8) &&
701 hasNoCarryFlagUses(
SDValue(U, 1)))
726 if (
U->getOperand(0).getOpcode() ==
ISD::SHL &&
730 if (
U->getOperand(1).getOpcode() ==
ISD::SHL &&
738 auto *
C = dyn_cast<ConstantSDNode>(U0.
getOperand(0));
739 if (
C &&
C->getSExtValue() == -2)
744 auto *
C = dyn_cast<ConstantSDNode>(U1.
getOperand(0));
745 if (
C &&
C->getSExtValue() == -2)
759 if (isa<ConstantSDNode>(
U->getOperand(1)))
780bool X86DAGToDAGISel::isProfitableToFormMaskedOp(
SDNode *
N)
const {
783 "Unexpected opcode!");
788 return N->getOperand(1).hasOneUse();
797 if (Chain.
getNode() == Load.getNode())
801 "Unexpected chain operand");
815 Load.getOperand(1), Load.getOperand(2));
819 Ops.
append(Call->op_begin() + 1, Call->op_end());
833 if (Callee.getNode() == Chain.
getNode() || !Callee.hasOneUse())
835 auto *LD = dyn_cast<LoadSDNode>(Callee.getNode());
853 if (isa<MemSDNode>(Chain.
getNode()) &&
854 cast<MemSDNode>(Chain.
getNode())->writeMem())
860 Callee.getValue(1).hasOneUse())
868 if ((Imm & 0x00FFFFFF) != 0x0F1EFA)
871 uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64,
872 0x65, 0x66, 0x67, 0xf0, 0xf2};
875 uint8_t Byte = (Imm >> i) & 0xFF;
887 return (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v64i8);
890void X86DAGToDAGISel::PreprocessISelDAG() {
891 bool MadeChange =
false;
893 E = CurDAG->allnodes_end();
I != E; ) {
912 MVT VT =
N->getSimpleValueType(0);
913 int64_t
Imm = cast<ConstantSDNode>(
N)->getSExtValue();
914 int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB;
921 SDValue Complement = CurDAG->getConstant(~Imm, dl, VT,
false,
true);
922 Complement = CurDAG->getNOT(dl, Complement, VT);
924 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Complement);
934 if (
N->getOpcode() ==
X86ISD::AND && !
N->hasAnyUseOfValue(1)) {
936 N->getOperand(0),
N->getOperand(1));
938 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
962 auto mayPreventLoadFold = [&]() {
964 N->getOpcode() ==
ISD::ADD && Subtarget->hasAVX() &&
965 !
N->getOperand(1).hasOneUse();
968 N->getSimpleValueType(0).isVector() && !mayPreventLoadFold()) {
974 MVT VT =
N->getSimpleValueType(0);
982 CurDAG->getNode(NewOpcode,
DL, VT,
N->getOperand(0),
AllOnes);
984 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
991 switch (
N->getOpcode()) {
993 MVT VT =
N->getSimpleValueType(0);
995 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1002 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1005 CurDAG->getIntPtrConstant(
Index, dl));
1008 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1017 MVT VT =
N->getSimpleValueType(0);
1019 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1021 auto *MemNode = cast<MemSDNode>(
N);
1023 SDVTList VTs = CurDAG->getVTList(NarrowVT, MVT::Other);
1024 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()};
1025 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode(
1027 MemNode->getMemOperand());
1030 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1033 CurDAG->getIntPtrConstant(
Index, dl));
1037 CurDAG->ReplaceAllUsesWith(
N, To);
1048 auto *Ld = cast<LoadSDNode>(
N);
1049 MVT VT =
N->getSimpleValueType(0);
1057 SDValue Chain = Ld->getChain();
1059 auto *UserLd = dyn_cast<LoadSDNode>(
User);
1060 MVT UserVT =
User->getSimpleValueType(0);
1062 UserLd->getBasePtr() ==
Ptr && UserLd->getChain() == Chain &&
1063 !
User->hasAnyUseOfValue(1) &&
1077 CurDAG->getIntPtrConstant(0, dl));
1078 SDValue Res = CurDAG->getBitcast(VT, Extract);
1082 CurDAG->ReplaceAllUsesWith(
N, To);
1091 EVT EleVT =
N->getOperand(0).getValueType().getVectorElementType();
1092 if (EleVT == MVT::i1)
1095 assert(Subtarget->hasSSE41() &&
"Expected SSE4.1 support!");
1096 assert(
N->getValueType(0).getVectorElementType() != MVT::i16 &&
1097 "We can't replace VSELECT with BLENDV in vXi16!");
1099 if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(
N->getOperand(0)) ==
1102 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
1103 CurDAG->getTargetConstant(0xCA,
SDLoc(
N), MVT::i8));
1106 N->getOperand(0),
N->getOperand(1),
1110 CurDAG->ReplaceAllUsesWith(
N,
R.getNode());
1123 if (!
N->getSimpleValueType(0).isVector())
1127 switch (
N->getOpcode()) {
1137 if (
N->isStrictFPOpcode())
1139 CurDAG->getNode(NewOpc,
SDLoc(
N), {
N->getValueType(0), MVT::Other},
1140 {
N->getOperand(0),
N->getOperand(1)});
1143 CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1146 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1156 if (!
N->getValueType(0).isVector())
1160 switch (
N->getOpcode()) {
1166 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1167 N->getOperand(0),
N->getOperand(1));
1169 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1178 if (!
N->getValueType(0).isVector())
1182 if (
N->getOperand(0).getScalarValueSizeInBits() == 1) {
1184 "Unexpected opcode for mask vector!");
1192 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1195 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1215 switch (
N->getOpcode()) {
1231 bool IsStrict =
N->isStrictFPOpcode();
1235 {
N->getValueType(0), MVT::Other},
1236 {
N->getOperand(0),
N->getOperand(1),
1237 CurDAG->getTargetConstant(Imm, dl, MVT::i32)});
1241 CurDAG->getTargetConstant(Imm, dl, MVT::i32));
1243 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1254 MVT VT =
N->getSimpleValueType(0);
1255 if (VT.
isVector() || VT == MVT::f128)
1258 MVT VecVT = VT == MVT::f64 ? MVT::v2f64
1259 : VT == MVT::f32 ? MVT::v4f32
1269 if (Subtarget->hasSSE2()) {
1274 switch (
N->getOpcode()) {
1281 Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1);
1284 Res = CurDAG->getNode(
N->getOpcode(), dl, VecVT, Op0, Op1);
1287 CurDAG->getIntPtrConstant(0, dl));
1289 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1296 if (OptLevel != CodeGenOptLevel::None &&
1299 !Subtarget->useIndirectThunkCalls() &&
1300 ((
N->getOpcode() ==
X86ISD::CALL && !Subtarget->slowTwoMemOps()) ||
1302 (Subtarget->is64Bit() ||
1303 !getTargetMachine().isPositionIndependent())))) {
1342 switch (
N->getOpcode()) {
1347 MVT SrcVT =
N->getOperand(0).getSimpleValueType();
1348 MVT DstVT =
N->getSimpleValueType(0);
1360 if (SrcIsSSE && DstIsSSE)
1363 if (!SrcIsSSE && !DstIsSSE) {
1368 if (
N->getConstantOperandVal(1))
1376 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1377 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1385 CurDAG->getEntryNode(), dl,
N->getOperand(0), MemTmp, MPI, MemVT);
1387 MemTmp, MPI, MemVT);
1394 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Result);
1403 MVT SrcVT =
N->getOperand(1).getSimpleValueType();
1404 MVT DstVT =
N->getSimpleValueType(0);
1416 if (SrcIsSSE && DstIsSSE)
1419 if (!SrcIsSSE && !DstIsSSE) {
1424 if (
N->getConstantOperandVal(2))
1432 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1433 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1443 SDVTList VTs = CurDAG->getVTList(MVT::Other);
1444 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), MemTmp};
1448 if (
N->getFlags().hasNoFPExcept()) {
1450 Flags.setNoFPExcept(
true);
1451 Store->setFlags(Flags);
1454 assert(SrcVT == MemVT &&
"Unexpected VT!");
1455 Store = CurDAG->getStore(
N->getOperand(0), dl,
N->getOperand(1), MemTmp,
1460 SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other);
1462 Result = CurDAG->getMemIntrinsicNode(
1465 if (
N->getFlags().hasNoFPExcept()) {
1467 Flags.setNoFPExcept(
true);
1471 assert(DstVT == MemVT &&
"Unexpected VT!");
1472 Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI);
1480 CurDAG->ReplaceAllUsesWith(
N,
Result.getNode());
1494 CurDAG->RemoveDeadNodes();
1498bool X86DAGToDAGISel::tryOptimizeRem8Extend(
SDNode *
N) {
1499 unsigned Opc =
N->getMachineOpcode();
1500 if (Opc != X86::MOVZX32rr8 && Opc != X86::MOVSX32rr8 &&
1501 Opc != X86::MOVSX64rr8)
1513 unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX
1514 : X86::MOVSX32rr8_NOREX;
1519 if (Opc == X86::MOVSX64rr8) {
1524 ReplaceUses(
N, Extend);
1533void X86DAGToDAGISel::PostprocessISelDAG() {
1535 if (
TM.getOptLevel() == CodeGenOptLevel::None)
1540 bool MadeChange =
false;
1541 while (Position != CurDAG->allnodes_begin()) {
1544 if (
N->use_empty() || !
N->isMachineOpcode())
1547 if (tryOptimizeRem8Extend(
N)) {
1554 unsigned Opc =
N->getMachineOpcode();
1555 if ((Opc == X86::TEST8rr || Opc == X86::TEST16rr ||
1556 Opc == X86::TEST32rr || Opc == X86::TEST64rr) &&
1557 N->getOperand(0) ==
N->getOperand(1) &&
1558 N->getOperand(0)->hasNUsesOfValue(2,
N->getOperand(0).getResNo()) &&
1559 N->getOperand(0).isMachineOpcode()) {
1561 unsigned N0Opc =
And.getMachineOpcode();
1562 if ((N0Opc == X86::AND8rr || N0Opc == X86::AND16rr ||
1563 N0Opc == X86::AND32rr || N0Opc == X86::AND64rr) &&
1564 !
And->hasAnyUseOfValue(1)) {
1569 ReplaceUses(
N,
Test);
1573 if ((N0Opc == X86::AND8rm || N0Opc == X86::AND16rm ||
1574 N0Opc == X86::AND32rm || N0Opc == X86::AND64rm) &&
1575 !
And->hasAnyUseOfValue(1)) {
1578 case X86::AND8rm: NewOpc = X86::TEST8mr;
break;
1579 case X86::AND16rm: NewOpc = X86::TEST16mr;
break;
1580 case X86::AND32rm: NewOpc = X86::TEST32mr;
break;
1581 case X86::AND64rm: NewOpc = X86::TEST64mr;
break;
1591 And.getOperand(6) };
1593 MVT::i32, MVT::Other, Ops);
1594 CurDAG->setNodeMemRefs(
1595 Test, cast<MachineSDNode>(
And.getNode())->memoperands());
1607 if ((Opc == X86::KORTESTBrr || Opc == X86::KORTESTWrr ||
1608 Opc == X86::KORTESTDrr || Opc == X86::KORTESTQrr) &&
1609 N->getOperand(0) ==
N->getOperand(1) &&
1610 N->isOnlyUserOf(
N->getOperand(0).getNode()) &&
1611 N->getOperand(0).isMachineOpcode() &&
1614 unsigned N0Opc =
And.getMachineOpcode();
1617 if (N0Opc == X86::KANDBrr ||
1618 (N0Opc == X86::KANDWrr && Subtarget->hasDQI()) ||
1619 N0Opc == X86::KANDDrr || N0Opc == X86::KANDQrr) {
1623 case X86::KORTESTBrr: NewOpc = X86::KTESTBrr;
break;
1624 case X86::KORTESTWrr: NewOpc = X86::KTESTWrr;
break;
1625 case X86::KORTESTDrr: NewOpc = X86::KTESTDrr;
break;
1626 case X86::KORTESTQrr: NewOpc = X86::KTESTQrr;
break;
1632 ReplaceUses(
N, KTest);
1639 if (Opc != TargetOpcode::SUBREG_TO_REG)
1642 unsigned SubRegIdx =
N->getConstantOperandVal(2);
1643 if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm)
1654 case X86::VMOVAPDrr:
case X86::VMOVUPDrr:
1655 case X86::VMOVAPSrr:
case X86::VMOVUPSrr:
1656 case X86::VMOVDQArr:
case X86::VMOVDQUrr:
1657 case X86::VMOVAPDYrr:
case X86::VMOVUPDYrr:
1658 case X86::VMOVAPSYrr:
case X86::VMOVUPSYrr:
1659 case X86::VMOVDQAYrr:
case X86::VMOVDQUYrr:
1660 case X86::VMOVAPDZ128rr:
case X86::VMOVUPDZ128rr:
1661 case X86::VMOVAPSZ128rr:
case X86::VMOVUPSZ128rr:
1662 case X86::VMOVDQA32Z128rr:
case X86::VMOVDQU32Z128rr:
1663 case X86::VMOVDQA64Z128rr:
case X86::VMOVDQU64Z128rr:
1664 case X86::VMOVAPDZ256rr:
case X86::VMOVUPDZ256rr:
1665 case X86::VMOVAPSZ256rr:
case X86::VMOVUPSZ256rr:
1666 case X86::VMOVDQA32Z256rr:
case X86::VMOVDQU32Z256rr:
1667 case X86::VMOVDQA64Z256rr:
case X86::VMOVDQU64Z256rr:
1672 if (!
In.isMachineOpcode() ||
1673 In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END)
1678 uint64_t TSFlags = getInstrInfo()->get(
In.getMachineOpcode()).TSFlags;
1686 CurDAG->UpdateNodeOperands(
N,
N->getOperand(0), In,
N->getOperand(2));
1691 CurDAG->RemoveDeadNodes();
1696void X86DAGToDAGISel::emitSpecialCodeForMain() {
1697 if (Subtarget->isTargetCygMing()) {
1699 auto &
DL = CurDAG->getDataLayout();
1702 CLI.setChain(CurDAG->getRoot())
1704 CurDAG->getExternalSymbol(
"__main", TLI->getPointerTy(
DL)),
1708 CurDAG->setRoot(
Result.second);
1712void X86DAGToDAGISel::emitFunctionEntryCode() {
1715 if (
F.hasExternalLinkage() &&
F.getName() ==
"main")
1716 emitSpecialCodeForMain();
1726 return isInt<31>(Val);
1730 X86ISelAddressMode &AM) {
1735 int64_t Val = AM.Disp +
Offset;
1738 if (Val != 0 && (AM.ES || AM.MCSym))
1742 if (Subtarget->is64Bit()) {
1745 AM.hasSymbolicDisplacement()))
1749 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
1768 if (Subtarget->isTarget64BitILP32() && !isUInt<31>(Val) &&
1769 !AM.hasBaseOrIndexReg())
1776bool X86DAGToDAGISel::matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
1777 bool AllowSegmentRegForX32) {
1789 if (
isNullConstant(Address) && AM.Segment.getNode() ==
nullptr &&
1790 !IndirectTlsSegRefs &&
1791 (Subtarget->isTargetGlibc() || Subtarget->isTargetAndroid() ||
1792 Subtarget->isTargetFuchsia())) {
1793 if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32)
1795 switch (
N->getPointerInfo().getAddrSpace()) {
1797 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
1800 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
1813bool X86DAGToDAGISel::matchWrapper(
SDValue N, X86ISelAddressMode &AM) {
1816 if (AM.hasSymbolicDisplacement())
1819 bool IsRIPRelTLS =
false;
1837 if (IsRIPRel && AM.hasBaseOrIndexReg())
1841 X86ISelAddressMode Backup = AM;
1845 if (
auto *
G = dyn_cast<GlobalAddressSDNode>(N0)) {
1846 AM.GV =
G->getGlobal();
1847 AM.SymbolFlags =
G->getTargetFlags();
1849 }
else if (
auto *CP = dyn_cast<ConstantPoolSDNode>(N0)) {
1850 AM.CP =
CP->getConstVal();
1851 AM.Alignment =
CP->getAlign();
1852 AM.SymbolFlags =
CP->getTargetFlags();
1854 }
else if (
auto *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
1855 AM.ES = S->getSymbol();
1856 AM.SymbolFlags = S->getTargetFlags();
1857 }
else if (
auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
1858 AM.MCSym = S->getMCSymbol();
1859 }
else if (
auto *J = dyn_cast<JumpTableSDNode>(N0)) {
1860 AM.JT = J->getIndex();
1861 AM.SymbolFlags = J->getTargetFlags();
1862 }
else if (
auto *BA = dyn_cast<BlockAddressSDNode>(N0)) {
1863 AM.BlockAddr = BA->getBlockAddress();
1864 AM.SymbolFlags = BA->getTargetFlags();
1865 Offset = BA->getOffset();
1870 if (Subtarget->is64Bit() && !IsRIPRel && AM.GV &&
1871 TM.isLargeGlobalValue(AM.GV)) {
1876 if (foldOffsetIntoAddress(
Offset, AM)) {
1882 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64));
1890bool X86DAGToDAGISel::matchAddress(
SDValue N, X86ISelAddressMode &AM) {
1891 if (matchAddressRecursively(
N, AM, 0))
1898 if (Subtarget->isTarget64BitILP32() &&
1899 AM.BaseType == X86ISelAddressMode::RegBase &&
1900 AM.Base_Reg.getNode() !=
nullptr && AM.IndexReg.getNode() ==
nullptr) {
1901 SDValue Save_Base_Reg = AM.Base_Reg;
1902 if (
auto *LoadN = dyn_cast<LoadSDNode>(Save_Base_Reg)) {
1904 if (matchLoadInAddress(LoadN, AM,
true))
1905 AM.Base_Reg = Save_Base_Reg;
1911 if (AM.Scale == 2 &&
1912 AM.BaseType == X86ISelAddressMode::RegBase &&
1913 AM.Base_Reg.getNode() ==
nullptr) {
1914 AM.Base_Reg = AM.IndexReg;
1921 (!AM.GV || !
TM.isLargeGlobalValue(AM.GV)) && Subtarget->is64Bit() &&
1922 AM.Scale == 1 && AM.BaseType == X86ISelAddressMode::RegBase &&
1923 AM.Base_Reg.getNode() ==
nullptr && AM.IndexReg.getNode() ==
nullptr &&
1925 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64);
1931bool X86DAGToDAGISel::matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
1937 X86ISelAddressMode Backup = AM;
1938 if (!matchAddressRecursively(
N.getOperand(0), AM,
Depth+1) &&
1939 !matchAddressRecursively(Handle.getValue().getOperand(1), AM,
Depth+1))
1944 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM,
1946 !matchAddressRecursively(Handle.getValue().getOperand(0), AM,
Depth + 1))
1953 if (AM.BaseType == X86ISelAddressMode::RegBase &&
1954 !AM.Base_Reg.getNode() &&
1955 !AM.IndexReg.getNode()) {
1956 N = Handle.getValue();
1957 AM.Base_Reg =
N.getOperand(0);
1958 AM.IndexReg =
N.getOperand(1);
1962 N = Handle.getValue();
1972 if (
N->getNodeId() == -1 ||
1992 X86ISelAddressMode &AM) {
1999 if (ScaleLog <= 0 || ScaleLog >= 4 ||
2000 Mask != (0xffu << ScaleLog))
2003 MVT XVT =
X.getSimpleValueType();
2004 MVT VT =
N.getSimpleValueType();
2029 AM.Scale = (1 << ScaleLog);
2037 X86ISelAddressMode &AM) {
2043 int64_t Mask = cast<ConstantSDNode>(
N->getOperand(1))->getSExtValue();
2048 bool FoundAnyExtend =
false;
2052 FoundAnyExtend =
true;
2070 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3)
2073 MVT VT =
N.getSimpleValueType();
2075 if (FoundAnyExtend) {
2096 AM.Scale = 1 << ShiftAmt;
2097 AM.IndexReg = NewAnd;
2131 X86ISelAddressMode &AM) {
2137 unsigned MaskIdx, MaskLen;
2140 unsigned MaskLZ = 64 - (MaskIdx + MaskLen);
2146 unsigned AMShiftAmt = MaskIdx;
2150 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2154 unsigned ScaleDown = (64 -
X.getSimpleValueType().getSizeInBits()) + ShiftAmt;
2155 if (MaskLZ < ScaleDown)
2157 MaskLZ -= ScaleDown;
2165 bool ReplacingAnyExtend =
false;
2167 unsigned ExtendBits =
X.getSimpleValueType().getSizeInBits() -
2168 X.getOperand(0).getSimpleValueType().getSizeInBits();
2171 X =
X.getOperand(0);
2172 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits;
2173 ReplacingAnyExtend =
true;
2175 APInt MaskedHighBits =
2182 MVT VT =
N.getSimpleValueType();
2183 if (ReplacingAnyExtend) {
2184 assert(
X.getValueType() != VT);
2191 MVT XVT =
X.getSimpleValueType();
2212 AM.Scale = 1 << AMShiftAmt;
2213 AM.IndexReg = NewExt;
2223 X86ISelAddressMode &AM,
2231 if (!Subtarget.hasTBM() &&
2232 !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR()))
2236 unsigned MaskIdx, MaskLen;
2244 unsigned AMShiftAmt = MaskIdx;
2248 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2250 MVT XVT =
X.getSimpleValueType();
2251 MVT VT =
N.getSimpleValueType();
2276 AM.Scale = 1 << AMShiftAmt;
2277 AM.IndexReg = NewExt;
2284 X86ISelAddressMode &AM,
2286 assert(AM.IndexReg.getNode() ==
nullptr &&
"IndexReg already matched");
2287 assert((AM.Scale == 1 || AM.Scale == 2 || AM.Scale == 4 || AM.Scale == 8) &&
2288 "Illegal index scale");
2294 EVT VT =
N.getValueType();
2295 unsigned Opc =
N.getOpcode();
2298 if (CurDAG->isBaseWithConstantOffset(
N)) {
2299 auto *AddVal = cast<ConstantSDNode>(
N.getOperand(1));
2301 if (!foldOffsetIntoAddress(
Offset, AM))
2302 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2306 if (Opc ==
ISD::ADD &&
N.getOperand(0) ==
N.getOperand(1)) {
2307 if (AM.Scale <= 4) {
2309 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2315 uint64_t ShiftAmt =
N.getConstantOperandVal(1);
2316 uint64_t ScaleAmt = 1ULL << ShiftAmt;
2317 if ((AM.Scale * ScaleAmt) <= 8) {
2318 AM.Scale *= ScaleAmt;
2319 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2327 if (Src.getOpcode() ==
ISD::ADD && Src->getFlags().hasNoSignedWrap() &&
2329 if (CurDAG->isBaseWithConstantOffset(Src)) {
2330 SDValue AddSrc = Src.getOperand(0);
2331 auto *AddVal = cast<ConstantSDNode>(Src.getOperand(1));
2333 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2335 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2341 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2342 CurDAG->RemoveDeadNode(
N.getNode());
2354 unsigned SrcOpc = Src.getOpcode();
2355 if (((SrcOpc ==
ISD::ADD && Src->getFlags().hasNoUnsignedWrap()) ||
2356 CurDAG->isADDLike(Src)) &&
2358 if (CurDAG->isBaseWithConstantOffset(Src)) {
2359 SDValue AddSrc = Src.getOperand(0);
2361 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2372 if ((AM.Scale * ScaleAmt) <= 8 &&
2374 CurDAG->MaskedValueIsZero(ShVal, HiBits))) {
2375 AM.Scale *= ScaleAmt;
2376 SDValue ExtShVal = CurDAG->getNode(Opc,
DL, VT, ShVal);
2385 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2387 SDValue ExtAdd = CurDAG->getNode(SrcOpc,
DL, VT, ExtSrc, ExtVal);
2391 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2392 CurDAG->RemoveDeadNode(
N.getNode());
2393 return Res ? Res : ExtSrc;
2403bool X86DAGToDAGISel::matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
2407 dbgs() <<
"MatchAddress: ";
2412 return matchAddressBase(
N, AM);
2417 if (AM.isRIPRelative()) {
2421 if (!(AM.ES || AM.MCSym) && AM.JT != -1)
2424 if (
auto *Cst = dyn_cast<ConstantSDNode>(
N))
2425 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM))
2430 switch (
N.getOpcode()) {
2433 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
2434 if (
const auto *ESNode = dyn_cast<MCSymbolSDNode>(
N.getOperand(0))) {
2436 AM.MCSym = ESNode->getMCSymbol();
2442 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2443 if (!foldOffsetIntoAddress(Val, AM))
2450 if (!matchWrapper(
N, AM))
2455 if (!matchLoadInAddress(cast<LoadSDNode>(
N), AM))
2460 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2461 AM.Base_Reg.getNode() ==
nullptr &&
2463 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
2464 AM.Base_FrameIndex = cast<FrameIndexSDNode>(
N)->getIndex();
2470 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2473 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
2474 unsigned Val = CN->getZExtValue();
2479 if (Val == 1 || Val == 2 || Val == 3) {
2481 AM.Scale = 1 << Val;
2482 AM.IndexReg = matchIndexRecursively(ShVal, AM,
Depth + 1);
2490 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2494 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2495 "Unexpected value size!");
2504 if (!isa<ConstantSDNode>(
N.getOperand(1)) ||
2505 !isa<ConstantSDNode>(
And.getOperand(1)))
2507 uint64_t Mask =
And.getConstantOperandVal(1) >>
N.getConstantOperandVal(1);
2519 if (
N.getResNo() != 0)
break;
2524 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2525 AM.Base_Reg.getNode() ==
nullptr &&
2526 AM.IndexReg.getNode() ==
nullptr) {
2527 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
2528 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
2529 CN->getZExtValue() == 9) {
2530 AM.Scale =
unsigned(CN->getZExtValue())-1;
2541 auto *AddVal = cast<ConstantSDNode>(MulVal.
getOperand(1));
2542 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue();
2543 if (foldOffsetIntoAddress(Disp, AM))
2544 Reg =
N.getOperand(0);
2546 Reg =
N.getOperand(0);
2549 AM.IndexReg = AM.Base_Reg =
Reg;
2568 X86ISelAddressMode Backup = AM;
2569 if (matchAddressRecursively(
N.getOperand(0), AM,
Depth+1)) {
2570 N = Handle.getValue();
2574 N = Handle.getValue();
2576 if (AM.IndexReg.getNode() || AM.isRIPRelative()) {
2591 RHS.getOperand(0).getValueType() == MVT::i32))
2595 if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() &&
2596 !AM.Base_Reg.getNode()->hasOneUse()) ||
2597 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
2601 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
2602 ((AM.Disp != 0) && (Backup.Disp == 0)) +
2603 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
2615 AM.NegateIndex =
true;
2623 if (!CurDAG->isADDLike(
N))
2627 if (!matchAdd(
N, AM,
Depth))
2636 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2640 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2641 "Unexpected value size!");
2643 if (!isa<ConstantSDNode>(
N.getOperand(1)))
2646 if (
N.getOperand(0).getOpcode() ==
ISD::SRL) {
2675 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2685 AM.IndexReg =
Index;
2691 if (Src.getOpcode() ==
ISD::AND && Src.hasOneUse())
2692 if (
auto *MaskC = dyn_cast<ConstantSDNode>(Src.getOperand(1))) {
2693 Mask = MaskC->getAPIntValue();
2694 Src = Src.getOperand(0);
2697 if (Src.getOpcode() ==
ISD::SHL && Src.hasOneUse()) {
2699 SDValue ShlSrc = Src.getOperand(0);
2700 SDValue ShlAmt = Src.getOperand(1);
2701 auto *ShAmtC = dyn_cast<ConstantSDNode>(ShlAmt);
2704 unsigned ShAmtV = ShAmtC->getZExtValue();
2712 if (!Src->getFlags().hasNoUnsignedWrap() &&
2713 !CurDAG->MaskedValueIsZero(ShlSrc, HighZeros & Mask))
2721 MVT VT =
N.getSimpleValueType();
2725 if (!
Mask.isAllOnes()) {
2726 Res = CurDAG->getConstant(
Mask.lshr(ShAmtV),
DL, SrcVT);
2728 Res = CurDAG->getNode(
ISD::AND,
DL, SrcVT, ShlSrc, Res);
2735 CurDAG->ReplaceAllUsesWith(
N, NewShl);
2736 CurDAG->RemoveDeadNode(
N.getNode());
2739 AM.Scale = 1 << ShAmtV;
2743 AM.IndexReg = matchIndexRecursively(Zext, AM,
Depth + 1);
2747 if (Src.getOpcode() ==
ISD::SRL && !
Mask.isAllOnes()) {
2750 Src.getOperand(0), AM))
2755 Src.getOperand(0), AM))
2760 Src.getOperand(0), AM, *Subtarget))
2768 return matchAddressBase(
N, AM);
2773bool X86DAGToDAGISel::matchAddressBase(
SDValue N, X86ISelAddressMode &AM) {
2775 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
2777 if (!AM.IndexReg.getNode()) {
2788 AM.BaseType = X86ISelAddressMode::RegBase;
2793bool X86DAGToDAGISel::matchVectorAddressRecursively(
SDValue N,
2794 X86ISelAddressMode &AM,
2798 dbgs() <<
"MatchVectorAddress: ";
2803 return matchAddressBase(
N, AM);
2806 switch (
N.getOpcode()) {
2808 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2809 if (!foldOffsetIntoAddress(Val, AM))
2814 if (!matchWrapper(
N, AM))
2822 X86ISelAddressMode Backup = AM;
2823 if (!matchVectorAddressRecursively(
N.getOperand(0), AM,
Depth + 1) &&
2824 !matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2830 if (!matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2832 !matchVectorAddressRecursively(Handle.getValue().getOperand(0), AM,
2837 N = Handle.getValue();
2842 return matchAddressBase(
N, AM);
2848bool X86DAGToDAGISel::matchVectorAddress(
SDValue N, X86ISelAddressMode &AM) {
2849 return matchVectorAddressRecursively(
N, AM, 0);
2857 X86ISelAddressMode AM;
2863 AM.IndexReg = matchIndexRecursively(IndexOp, AM, 0);
2865 AM.IndexReg = IndexOp;
2869 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2871 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2873 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2879 if (matchVectorAddress(BasePtr, AM))
2882 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2896 X86ISelAddressMode AM;
2908 unsigned AddrSpace =
2909 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
2911 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2913 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2915 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2920 MVT VT =
N.getSimpleValueType();
2922 if (matchAddress(
N, AM))
2925 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2942 N =
N.getOperand(0);
2957 const GlobalValue *GV = cast<GlobalAddressSDNode>(
N)->getGlobal();
2959 return CR->getUnsignedMax().ult(1ull << 32);
2961 return !
TM.isLargeGlobalValue(GV);
2970 if (!selectLEAAddr(
N,
Base, Scale,
Index, Disp, Segment))
2973 auto *
RN = dyn_cast<RegisterSDNode>(
Base);
2974 if (RN &&
RN->getReg() == 0)
2975 Base = CurDAG->getRegister(0, MVT::i64);
2976 else if (
Base.getValueType() == MVT::i32 && !isa<FrameIndexSDNode>(
Base)) {
2980 Base = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
2984 RN = dyn_cast<RegisterSDNode>(
Index);
2985 if (RN &&
RN->getReg() == 0)
2986 Index = CurDAG->getRegister(0, MVT::i64);
2989 "Expect to be extending 32-bit registers for use in LEA");
2992 Index = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
3001bool X86DAGToDAGISel::selectLEAAddr(
SDValue N,
3005 X86ISelAddressMode AM;
3009 MVT VT =
N.getSimpleValueType();
3014 SDValue T = CurDAG->getRegister(0, MVT::i32);
3016 if (matchAddress(
N, AM))
3021 unsigned Complexity = 0;
3022 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode())
3024 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
3027 if (AM.IndexReg.getNode())
3040 if (AM.hasSymbolicDisplacement()) {
3042 if (Subtarget->is64Bit())
3052 auto isMathWithFlags = [](
SDValue V) {
3053 switch (
V.getOpcode()) {
3074 if (isMathWithFlags(
N.getOperand(0)) || isMathWithFlags(
N.getOperand(1)))
3082 if (Complexity <= 2)
3085 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
3096 X86ISelAddressMode AM;
3097 if (
auto *GA = dyn_cast<GlobalAddressSDNode>(
N)) {
3098 AM.GV = GA->getGlobal();
3099 AM.Disp += GA->getOffset();
3100 AM.SymbolFlags = GA->getTargetFlags();
3102 auto *SA = cast<ExternalSymbolSDNode>(
N);
3103 AM.ES = SA->getSymbol();
3104 AM.SymbolFlags = SA->getTargetFlags();
3107 if (Subtarget->is32Bit()) {
3109 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
3112 MVT VT =
N.getSimpleValueType();
3113 getAddressOperands(AM,
SDLoc(
N), VT,
Base, Scale,
Index, Disp, Segment);
3121 EVT VT =
N.getValueType();
3122 bool WasTruncated =
false;
3124 WasTruncated =
true;
3125 N =
N.getOperand(0);
3134 unsigned Opc =
N.getOperand(0)->getOpcode();
3136 Op =
N.getOperand(0);
3139 return !WasTruncated;
3143 auto *GA = cast<GlobalAddressSDNode>(
N.getOperand(0));
3144 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange();
3145 if (!CR || CR->getUnsignedMax().uge(1ull << VT.
getSizeInBits()))
3149 Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(),
SDLoc(
N), VT,
3150 GA->getOffset(), GA->getTargetFlags());
3158 assert(Root &&
P &&
"Unknown root/parent nodes");
3160 !IsProfitableToFold(
N,
P, Root) ||
3161 !IsLegalToFold(
N,
P, Root, OptLevel))
3164 return selectAddr(
N.getNode(),
3165 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3172 assert(Root &&
P &&
"Unknown root/parent nodes");
3174 !IsProfitableToFold(
N,
P, Root) ||
3175 !IsLegalToFold(
N,
P, Root, OptLevel))
3178 return selectAddr(
N.getNode(),
3179 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3185SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
3186 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
3188 return CurDAG->getRegister(GlobalBaseReg, TLI->
getPointerTy(
DL)).getNode();
3191bool X86DAGToDAGISel::isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const {
3193 N =
N->getOperand(0).getNode();
3197 auto *GA = dyn_cast<GlobalAddressSDNode>(
N->getOperand(0));
3201 auto *GV = GA->getGlobal();
3204 return CR->getSignedMin().sge(-1ull << Width) &&
3205 CR->getSignedMax().slt(1ull << Width);
3210 return Width == 32 && !
TM.isLargeGlobalValue(GV);
3214 assert(
N->isMachineOpcode() &&
"Unexpected node");
3215 unsigned Opc =
N->getMachineOpcode();
3216 const MCInstrDesc &MCID = getInstrInfo()->get(Opc);
3221 return static_cast<X86::CondCode>(
N->getConstantOperandVal(CondNo));
3226bool X86DAGToDAGISel::onlyUsesZeroFlag(
SDValue Flags)
const {
3231 if (UI.getUse().getResNo() !=
Flags.getResNo())
3235 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3239 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3241 if (FlagUI.getUse().getResNo() != 1)
continue;
3243 if (!FlagUI->isMachineOpcode())
return false;
3262bool X86DAGToDAGISel::hasNoSignFlagUses(
SDValue Flags)
const {
3267 if (UI.getUse().getResNo() !=
Flags.getResNo())
3271 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3275 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3277 if (FlagUI.getUse().getResNo() != 1)
continue;
3279 if (!FlagUI->isMachineOpcode())
return false;
3318 bool X86DAGToDAGISel::hasNoCarryFlagUses(
SDValue Flags)
const {
3323 if (UI.getUse().getResNo() !=
Flags.getResNo())
3326 unsigned UIOpc = UI->getOpcode();
3330 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3334 FlagUI != FlagUE; ++FlagUI) {
3336 if (FlagUI.getUse().getResNo() != 1)
3339 if (!FlagUI->isMachineOpcode())
3380 if (StoredVal.
getResNo() != 0)
return false;
3394 LoadNode = cast<LoadSDNode>(Load);
3397 if (!Load.hasOneUse())
3405 bool FoundLoad =
false;
3409 const unsigned int Max = 1024;
3451 if (Chain == Load.getValue(1)) {
3457 if (
Op == Load.getValue(1)) {
3473 if (
Op.getNode() != LoadNode)
3506bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(
SDNode *
Node) {
3507 auto *StoreNode = cast<StoreSDNode>(
Node);
3514 EVT MemVT = StoreNode->getMemoryVT();
3515 if (MemVT != MVT::i64 && MemVT != MVT::i32 && MemVT != MVT::i16 &&
3519 bool IsCommutable =
false;
3520 bool IsNegate =
false;
3534 IsCommutable =
true;
3538 unsigned LoadOpNo = IsNegate ? 1 : 0;
3542 LoadNode, InputChain)) {
3549 LoadNode, InputChain))
3558 auto SelectOpcode = [&](
unsigned Opc64,
unsigned Opc32,
unsigned Opc16,
3579 unsigned NewOpc = SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m,
3589 if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) {
3593 if ((IsOne || IsNegOne) && hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3596 ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
3597 : SelectOpcode(
X86::DEC64m,
X86::DEC32m,
X86::DEC16m,
X86::DEC8m);
3610 auto SelectRegOpcode = [SelectOpcode](
unsigned Opc) {
3613 return SelectOpcode(X86::ADD64mr, X86::ADD32mr, X86::ADD16mr,
3616 return SelectOpcode(X86::ADC64mr, X86::ADC32mr, X86::ADC16mr,
3619 return SelectOpcode(X86::SUB64mr, X86::SUB32mr, X86::SUB16mr,
3622 return SelectOpcode(X86::SBB64mr, X86::SBB32mr, X86::SBB16mr,
3625 return SelectOpcode(X86::AND64mr, X86::AND32mr, X86::AND16mr,
3628 return SelectOpcode(X86::OR64mr, X86::OR32mr, X86::OR16mr, X86::OR8mr);
3630 return SelectOpcode(X86::XOR64mr, X86::XOR32mr, X86::XOR16mr,
3636 auto SelectImmOpcode = [SelectOpcode](
unsigned Opc) {
3639 return SelectOpcode(X86::ADD64mi32, X86::ADD32mi, X86::ADD16mi,
3642 return SelectOpcode(X86::ADC64mi32, X86::ADC32mi, X86::ADC16mi,
3645 return SelectOpcode(X86::SUB64mi32, X86::SUB32mi, X86::SUB16mi,
3648 return SelectOpcode(X86::SBB64mi32, X86::SBB32mi, X86::SBB16mi,
3651 return SelectOpcode(X86::AND64mi32, X86::AND32mi, X86::AND16mi,
3654 return SelectOpcode(X86::OR64mi32, X86::OR32mi, X86::OR16mi,
3657 return SelectOpcode(X86::XOR64mi32, X86::XOR32mi, X86::XOR16mi,
3664 unsigned NewOpc = SelectRegOpcode(Opc);
3669 if (
auto *OperandC = dyn_cast<ConstantSDNode>(Operand)) {
3670 int64_t OperandV = OperandC->getSExtValue();
3676 ((MemVT != MVT::i8 && !isInt<8>(OperandV) && isInt<8>(-OperandV)) ||
3677 (MemVT == MVT::i64 && !isInt<32>(OperandV) &&
3678 isInt<32>(-OperandV))) &&
3679 hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3680 OperandV = -OperandV;
3684 if (MemVT != MVT::i64 || isInt<32>(OperandV)) {
3685 Operand = CurDAG->getTargetConstant(OperandV,
SDLoc(
Node), MemVT);
3686 NewOpc = SelectImmOpcode(Opc);
3692 CurDAG->getCopyToReg(InputChain,
SDLoc(
Node), X86::EFLAGS,
3696 Segment, Operand, CopyTo, CopyTo.
getValue(1)};
3697 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3701 Segment, Operand, InputChain};
3702 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3713 CurDAG->setNodeMemRefs(Result, MemOps);
3719 CurDAG->RemoveDeadNode(
Node);
3730bool X86DAGToDAGISel::matchBitExtract(
SDNode *
Node) {
3734 "Should be either an and-mask, or right-shift after clearing high bits.");
3737 if (!Subtarget->hasBMI() && !Subtarget->hasBMI2())
3740 MVT NVT =
Node->getSimpleValueType(0);
3743 if (NVT != MVT::i32 && NVT != MVT::i64)
3751 const bool AllowExtraUsesByDefault = Subtarget->hasBMI2();
3752 auto checkUses = [AllowExtraUsesByDefault](
3754 std::optional<bool> AllowExtraUses) {
3755 return AllowExtraUses.value_or(AllowExtraUsesByDefault) ||
3756 Op.getNode()->hasNUsesOfValue(NUses,
Op.getResNo());
3758 auto checkOneUse = [checkUses](
SDValue Op,
3759 std::optional<bool> AllowExtraUses =
3761 return checkUses(
Op, 1, AllowExtraUses);
3763 auto checkTwoUse = [checkUses](
SDValue Op,
3764 std::optional<bool> AllowExtraUses =
3766 return checkUses(
Op, 2, AllowExtraUses);
3769 auto peekThroughOneUseTruncation = [checkOneUse](
SDValue V) {
3771 assert(
V.getSimpleValueType() == MVT::i32 &&
3772 V.getOperand(0).getSimpleValueType() == MVT::i64 &&
3773 "Expected i64 -> i32 truncation");
3774 V =
V.getOperand(0);
3780 auto matchPatternA = [checkOneUse, peekThroughOneUseTruncation, &NBits,
3783 if (
Mask->getOpcode() !=
ISD::ADD || !checkOneUse(Mask))
3789 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3794 NBits =
M0->getOperand(1);
3795 NegateNBits =
false;
3799 auto isAllOnes = [
this, peekThroughOneUseTruncation, NVT](
SDValue V) {
3800 V = peekThroughOneUseTruncation(V);
3801 return CurDAG->MaskedValueIsAllOnes(
3807 auto matchPatternB = [checkOneUse, isAllOnes, peekThroughOneUseTruncation,
3810 if (
Mask.getOpcode() !=
ISD::XOR || !checkOneUse(Mask))
3813 if (!isAllOnes(
Mask->getOperand(1)))
3816 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3820 if (!isAllOnes(
M0->getOperand(0)))
3822 NBits =
M0->getOperand(1);
3823 NegateNBits =
false;
3829 auto canonicalizeShiftAmt = [&NBits, &NegateNBits](
SDValue ShiftAmt,
3830 unsigned Bitwidth) {
3835 NBits = NBits.getOperand(0);
3840 auto *V0 = dyn_cast<ConstantSDNode>(NBits.getOperand(0));
3841 if (!V0 || V0->getZExtValue() != Bitwidth)
3843 NBits = NBits.getOperand(1);
3844 NegateNBits =
false;
3850 auto matchPatternC = [checkOneUse, peekThroughOneUseTruncation, &NegateNBits,
3853 Mask = peekThroughOneUseTruncation(Mask);
3854 unsigned Bitwidth =
Mask.getSimpleValueType().getSizeInBits();
3856 if (
Mask.getOpcode() !=
ISD::SRL || !checkOneUse(Mask))
3863 if (!checkOneUse(
M1))
3865 canonicalizeShiftAmt(
M1, Bitwidth);
3870 return !NegateNBits;
3878 auto matchPatternD = [checkOneUse, checkTwoUse, canonicalizeShiftAmt,
3879 AllowExtraUsesByDefault, &NegateNBits,
3892 canonicalizeShiftAmt(N1, Bitwidth);
3896 const bool AllowExtraUses = AllowExtraUsesByDefault && !NegateNBits;
3897 if (!checkOneUse(N0, AllowExtraUses) || !checkTwoUse(N1, AllowExtraUses))
3903 auto matchLowBitMask = [matchPatternA, matchPatternB,
3905 return matchPatternA(Mask) || matchPatternB(Mask) || matchPatternC(Mask);
3909 X =
Node->getOperand(0);
3912 if (matchLowBitMask(Mask)) {
3916 if (!matchLowBitMask(Mask))
3920 X = CurDAG->getAllOnesConstant(
SDLoc(
Node), NVT);
3921 }
else if (!matchPatternD(
Node))
3926 if (NegateNBits && !Subtarget->hasBMI2())
3938 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, MVT::i32), 0);
3941 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit,
DL, MVT::i32);
3943 NBits =
SDValue(CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG,
DL,
3944 MVT::i32, ImplDef, NBits, SRIdxVal),
3954 NBits = CurDAG->getNode(
ISD::SUB,
DL, MVT::i32, BitWidthC, NBits);
3958 if (Subtarget->hasBMI2()) {
3960 if (NVT != MVT::i32) {
3968 SelectCode(Extract.
getNode());
3977 SDValue RealX = peekThroughOneUseTruncation(
X);
3983 MVT XVT =
X.getSimpleValueType();
3993 SDValue C8 = CurDAG->getConstant(8,
DL, MVT::i8);
4001 SDValue ShiftAmt =
X.getOperand(1);
4002 X =
X.getOperand(0);
4005 "Expected shift amount to be i8");
4009 SDValue OrigShiftAmt = ShiftAmt;
4014 Control = CurDAG->getNode(
ISD::OR,
DL, MVT::i32, Control, ShiftAmt);
4019 if (XVT != MVT::i32) {
4034 SelectCode(Extract.
getNode());
4041 MVT NVT =
Node->getSimpleValueType(0);
4054 Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR());
4055 if (!PreferBEXTR && !Subtarget->hasBMI2())
4067 if (NVT != MVT::i32 && NVT != MVT::i64)
4071 auto *MaskCst = dyn_cast<ConstantSDNode>(N1);
4072 auto *ShiftCst = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
4073 if (!MaskCst || !ShiftCst)
4081 uint64_t Shift = ShiftCst->getZExtValue();
4086 if (Shift == 8 && MaskSize == 8)
4097 if (!PreferBEXTR && MaskSize <= 32)
4101 unsigned ROpc, MOpc;
4103#define GET_EGPR_IF_ENABLED(OPC) (Subtarget->hasEGPR() ? OPC##_EVEX : OPC)
4105 assert(Subtarget->hasBMI2() &&
"We must have BMI2's BZHI then.");
4109 Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT);
4114 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4115 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4121 Control = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT);
4122 if (Subtarget->hasTBM()) {
4123 ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri;
4124 MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi;
4126 assert(Subtarget->hasBMI() &&
"We must have BMI1's BEXTR then.");
4132 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4133 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4139 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4140 if (tryFoldLoad(
Node, N0.
getNode(), Input, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4142 Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Control, Input.
getOperand(0)};
4143 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
4144 NewNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4148 CurDAG->setNodeMemRefs(NewNode, {cast<LoadSDNode>(Input)->getMemOperand()});
4150 NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, Control);
4155 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT);
4159 CurDAG->getMachineNode(NewOpc, dl, NVT,
SDValue(NewNode, 0), ShAmt);
4166MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
4167 bool MayFoldLoad,
const SDLoc &dl,
4172 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4173 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4176 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4177 if (MayFoldLoad && tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4178 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4180 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other);
4181 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4185 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
4190 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32);
4191 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4198MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
4199 bool MayFoldLoad,
const SDLoc &dl,
4205 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4206 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4209 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4210 if (MayFoldLoad && tryFoldLoad(
Node, N2, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4211 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4213 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other, MVT::Glue);
4214 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4219 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N2)->getMemOperand()});
4224 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Glue);
4225 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4230bool X86DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
4231 EVT VT =
N->getValueType(0);
4238 unsigned Size = VT == MVT::i64 ? 64 : 32;
4240 SDValue OrigShiftAmt =
N->getOperand(1);
4241 SDValue ShiftAmt = OrigShiftAmt;
4256 auto *Add0C = dyn_cast<ConstantSDNode>(Add0);
4257 auto *Add1C = dyn_cast<ConstantSDNode>(Add1);
4260 if (Add1C && Add1C->getAPIntValue().urem(
Size) == 0) {
4264 ((Add0C && Add0C->getAPIntValue().urem(
Size) ==
Size - 1) ||
4265 (Add1C && Add1C->getAPIntValue().urem(
Size) ==
Size - 1))) {
4269 assert(Add0C ==
nullptr || Add1C ==
nullptr);
4278 NewShiftAmt = CurDAG->getNode(
ISD::XOR,
DL, OpVT,
4279 Add0C ==
nullptr ? Add0 : Add1,
AllOnes);
4285 Add0C->getZExtValue() != 0) {
4288 if (Add0C->getZExtValue() %
Size == 0)
4291 Add0C->getZExtValue() % 32 == 0) {
4299 Add0 = CurDAG->getZExtOrTrunc(Add0,
DL, SubVT);
4303 X = CurDAG->getNode(
ISD::ADD,
DL, SubVT, Add1, Add0);
4325 NewShiftAmt = CurDAG->getNode(
ISD::TRUNCATE,
DL, MVT::i8, NewShiftAmt);
4332 NewShiftAmt = CurDAG->getNode(
ISD::AND,
DL, MVT::i8, NewShiftAmt,
4333 CurDAG->getConstant(
Size - 1,
DL, MVT::i8));
4337 SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(
N,
N->getOperand(0),
4339 if (UpdatedNode !=
N) {
4342 ReplaceNode(
N, UpdatedNode);
4349 CurDAG->RemoveDeadNode(OrigShiftAmt.
getNode());
4357bool X86DAGToDAGISel::tryShrinkShlLogicImm(
SDNode *
N) {
4358 MVT NVT =
N->getSimpleValueType(0);
4359 unsigned Opcode =
N->getOpcode();
4367 auto *Cst = dyn_cast<ConstantSDNode>(N1);
4371 int64_t Val = Cst->getSExtValue();
4376 bool FoundAnyExtend =
false;
4380 FoundAnyExtend =
true;
4388 if (NVT != MVT::i32 && NVT != MVT::i64)
4391 auto *ShlCst = dyn_cast<ConstantSDNode>(Shift.
getOperand(1));
4395 uint64_t ShAmt = ShlCst->getZExtValue();
4399 uint64_t RemovedBitsMask = (1ULL << ShAmt) - 1;
4400 if (Opcode !=
ISD::AND && (Val & RemovedBitsMask) != 0)
4405 auto CanShrinkImmediate = [&](int64_t &ShiftedVal) {
4409 ShiftedVal = (
uint64_t)Val >> ShAmt;
4410 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4413 if (ShiftedVal == UINT8_MAX || ShiftedVal == UINT16_MAX)
4416 ShiftedVal = Val >> ShAmt;
4417 if ((!isInt<8>(Val) && isInt<8>(ShiftedVal)) ||
4418 (!isInt<32>(Val) && isInt<32>(ShiftedVal)))
4422 ShiftedVal = (
uint64_t)Val >> ShAmt;
4423 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4430 if (!CanShrinkImmediate(ShiftedVal))
4440 unsigned ZExtWidth = Cst->getAPIntValue().getActiveBits();
4446 NeededMask &= ~Cst->getAPIntValue();
4448 if (CurDAG->MaskedValueIsZero(
N->getOperand(0), NeededMask))
4453 if (FoundAnyExtend) {
4459 SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT);
4461 SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT,
X, NewCst);
4470bool X86DAGToDAGISel::matchVPTERNLOG(
SDNode *Root,
SDNode *ParentA,
4474 assert(
A.isOperandOf(ParentA) &&
B.isOperandOf(ParentB) &&
4475 C.isOperandOf(ParentC) &&
"Incorrect parent node");
4477 auto tryFoldLoadOrBCast =
4480 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4486 L =
L.getOperand(0);
4493 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4494 unsigned Size = MemIntr->getMemoryVT().getSizeInBits();
4498 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4501 bool FoldedLoad =
false;
4502 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4503 if (tryFoldLoadOrBCast(Root, ParentC,
C, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4505 }
else if (tryFoldLoadOrBCast(Root, ParentA,
A, Tmp0, Tmp1, Tmp2, Tmp3,
4510 uint8_t OldImm =
Imm;
4511 Imm = OldImm & 0xa5;
4512 if (OldImm & 0x02)
Imm |= 0x10;
4513 if (OldImm & 0x10)
Imm |= 0x02;
4514 if (OldImm & 0x08)
Imm |= 0x40;
4515 if (OldImm & 0x40)
Imm |= 0x08;
4516 }
else if (tryFoldLoadOrBCast(Root, ParentB,
B, Tmp0, Tmp1, Tmp2, Tmp3,
4521 uint8_t OldImm =
Imm;
4522 Imm = OldImm & 0x99;
4523 if (OldImm & 0x02)
Imm |= 0x04;
4524 if (OldImm & 0x04)
Imm |= 0x02;
4525 if (OldImm & 0x20)
Imm |= 0x40;
4526 if (OldImm & 0x40)
Imm |= 0x20;
4531 SDValue TImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
4537 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
4541 auto *MemIntr = cast<MemIntrinsicSDNode>(
C);
4542 unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits();
4543 assert((EltSize == 32 || EltSize == 64) &&
"Unexpected broadcast size!");
4545 bool UseD = EltSize == 32;
4547 Opc = UseD ? X86::VPTERNLOGDZ128rmbi : X86::VPTERNLOGQZ128rmbi;
4549 Opc = UseD ? X86::VPTERNLOGDZ256rmbi : X86::VPTERNLOGQZ256rmbi;
4551 Opc = UseD ? X86::VPTERNLOGDZrmbi : X86::VPTERNLOGQZrmbi;
4557 Opc = UseD ? X86::VPTERNLOGDZ128rmi : X86::VPTERNLOGQZ128rmi;
4559 Opc = UseD ? X86::VPTERNLOGDZ256rmi : X86::VPTERNLOGQZ256rmi;
4561 Opc = UseD ? X86::VPTERNLOGDZrmi : X86::VPTERNLOGQZrmi;
4566 SDValue Ops[] = {
A,
B, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, TImm,
C.getOperand(0)};
4567 MNode = CurDAG->getMachineNode(Opc,
DL, VTs, Ops);
4570 ReplaceUses(
C.getValue(1),
SDValue(MNode, 1));
4572 CurDAG->setNodeMemRefs(MNode, {cast<MemSDNode>(
C)->getMemOperand()});
4577 Opc = UseD ? X86::VPTERNLOGDZ128rri : X86::VPTERNLOGQZ128rri;
4579 Opc = UseD ? X86::VPTERNLOGDZ256rri : X86::VPTERNLOGQZ256rri;
4581 Opc = UseD ? X86::VPTERNLOGDZrri : X86::VPTERNLOGQZrri;
4585 MNode = CurDAG->getMachineNode(Opc,
DL, NVT, {
A,
B,
C, TImm});
4589 CurDAG->RemoveDeadNode(Root);
4595bool X86DAGToDAGISel::tryVPTERNLOG(
SDNode *
N) {
4596 MVT NVT =
N->getSimpleValueType(0);
4599 if (!NVT.
isVector() || !Subtarget->hasAVX512() ||
4610 auto getFoldableLogicOp = [](
SDValue Op) {
4613 Op =
Op.getOperand(0);
4615 if (!
Op.hasOneUse())
4618 unsigned Opc =
Op.getOpcode();
4627 if ((FoldableOp = getFoldableLogicOp(N1))) {
4629 }
else if ((FoldableOp = getFoldableLogicOp(N0))) {
4642 uint8_t TernlogMagicA = 0xf0;
4643 uint8_t TernlogMagicB = 0xcc;
4644 uint8_t TernlogMagicC = 0xaa;
4653 Parent =
Op.getNode();
4654 Op =
Op.getOperand(0);
4658 PeekThroughNot(
A, ParentA, TernlogMagicA);
4659 PeekThroughNot(
B, ParentB, TernlogMagicB);
4660 PeekThroughNot(
C, ParentC, TernlogMagicC);
4665 case ISD::AND:
Imm = TernlogMagicB & TernlogMagicC;
break;
4666 case ISD::OR:
Imm = TernlogMagicB | TernlogMagicC;
break;
4667 case ISD::XOR:
Imm = TernlogMagicB ^ TernlogMagicC;
break;
4671 switch (
N->getOpcode()) {
4675 Imm &= ~TernlogMagicA;
4677 Imm = ~(
Imm) & TernlogMagicA;
4684 return matchVPTERNLOG(
N, ParentA, ParentB, ParentC,
A,
B,
C, Imm);
4694bool X86DAGToDAGISel::shrinkAndImmediate(
SDNode *
And) {
4697 MVT VT =
And->getSimpleValueType(0);
4698 if (VT != MVT::i32 && VT != MVT::i64)
4701 auto *And1C = dyn_cast<ConstantSDNode>(
And->getOperand(1));
4710 APInt MaskVal = And1C->getAPIntValue();
4712 if (!MaskLZ || (VT == MVT::i64 && MaskLZ == 32))
4716 if (VT == MVT::i64 && MaskLZ >= 32) {
4718 MaskVal = MaskVal.
trunc(32);
4723 APInt NegMaskVal = MaskVal | HighZeros;
4732 if (VT == MVT::i64 && MaskVal.
getBitWidth() < 64) {
4733 NegMaskVal = NegMaskVal.
zext(64);
4734 HighZeros = HighZeros.
zext(64);
4739 if (!CurDAG->MaskedValueIsZero(And0, HighZeros))
4759 bool FoldedBCast,
bool Masked) {
4760#define VPTESTM_CASE(VT, SUFFIX) \
4763 return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
4764 return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
4767#define VPTESTM_BROADCAST_CASES(SUFFIX) \
4768default: llvm_unreachable("Unexpected VT!"); \
4769VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
4770VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
4771VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
4772VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
4773VPTESTM_CASE(v16i32, DZ##SUFFIX) \
4774VPTESTM_CASE(v8i64, QZ##SUFFIX)
4776#define VPTESTM_FULL_CASES(SUFFIX) \
4777VPTESTM_BROADCAST_CASES(SUFFIX) \
4778VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
4779VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
4780VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
4781VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
4782VPTESTM_CASE(v64i8, BZ##SUFFIX) \
4783VPTESTM_CASE(v32i16, WZ##SUFFIX)
4801#undef VPTESTM_FULL_CASES
4802#undef VPTESTM_BROADCAST_CASES
4808bool X86DAGToDAGISel::tryVPTESTM(
SDNode *Root,
SDValue Setcc,
4810 assert(Subtarget->hasAVX512() &&
"Expected AVX512!");
4860 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4865 if (CmpSVT != MVT::i32 && CmpSVT != MVT::i64)
4871 L =
L.getOperand(0);
4877 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4878 if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.
getSizeInBits())
4881 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4885 bool CanFoldLoads = Src0 != Src1;
4887 bool FoldedLoad =
false;
4888 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4890 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src1, Tmp0, Tmp1, Tmp2,
4894 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src0, Tmp0, Tmp1,
4903 bool IsMasked = InMask.
getNode() !=
nullptr;
4916 SDValue ImplDef =
SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl,
4918 Src0 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src0);
4921 Src1 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src1);
4926 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4927 InMask =
SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4928 dl, MaskVT, InMask, RC), 0);
4933 unsigned Opc =
getVPTESTMOpc(CmpVT, IsTestN, FoldedLoad, FoldedBCast,
4938 SDVTList VTs = CurDAG->getVTList(MaskVT, MVT::Other);
4941 SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4943 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
4945 SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4947 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
4953 CurDAG->setNodeMemRefs(CNode, {cast<MemSDNode>(Src1)->getMemOperand()});
4956 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, InMask, Src0, Src1);
4958 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, Src0, Src1);
4964 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4965 CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4966 dl, ResVT,
SDValue(CNode, 0), RC);
4970 CurDAG->RemoveDeadNode(Root);
4976bool X86DAGToDAGISel::tryMatchBitSelect(
SDNode *
N) {
4979 MVT NVT =
N->getSimpleValueType(0);
4982 if (!NVT.
isVector() || !Subtarget->hasAVX512())
5016 SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8);
5025 MVT NVT =
Node->getSimpleValueType(0);
5026 unsigned Opcode =
Node->getOpcode();
5029 if (
Node->isMachineOpcode()) {
5031 Node->setNodeId(-1);
5038 unsigned IntNo =
Node->getConstantOperandVal(1);
5041 case Intrinsic::x86_encodekey128:
5042 case Intrinsic::x86_encodekey256: {
5043 if (!Subtarget->hasKL())
5049 case Intrinsic::x86_encodekey128:
5052 case Intrinsic::x86_encodekey256:
5058 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(3),
5060 if (Opcode == X86::ENCODEKEY256 || Opcode == X86::ENCODEKEY256_EVEX)
5061 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(4),
5065 Opcode, dl,
Node->getVTList(),
5066 {Node->getOperand(2), Chain, Chain.getValue(1)});
5067 ReplaceNode(
Node, Res);
5070 case Intrinsic::x86_tileloadd64_internal:
5071 case Intrinsic::x86_tileloaddt164_internal: {
5072 if (!Subtarget->hasAMXTILE())
5074 unsigned Opc = IntNo == Intrinsic::x86_tileloadd64_internal
5076 : X86::PTILELOADDT1V;
5079 SDValue Scale = getI8Imm(1, dl);
5081 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5082 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5086 Node->getOperand(3),
5093 CNode = CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops);
5094 ReplaceNode(
Node, CNode);
5101 unsigned IntNo =
Node->getConstantOperandVal(1);
5104 case Intrinsic::x86_sse3_monitor:
5105 case Intrinsic::x86_monitorx:
5106 case Intrinsic::x86_clzero: {
5107 bool Use64BitPtr =
Node->getOperand(2).getValueType() == MVT::i64;
5112 case Intrinsic::x86_sse3_monitor:
5113 if (!Subtarget->hasSSE3())
5115 Opc = Use64BitPtr ? X86::MONITOR64rrr : X86::MONITOR32rrr;
5117 case Intrinsic::x86_monitorx:
5118 if (!Subtarget->hasMWAITX())
5120 Opc = Use64BitPtr ? X86::MONITORX64rrr : X86::MONITORX32rrr;
5122 case Intrinsic::x86_clzero:
5123 if (!Subtarget->hasCLZERO())
5125 Opc = Use64BitPtr ? X86::CLZERO64r : X86::CLZERO32r;
5130 unsigned PtrReg = Use64BitPtr ? X86::RAX : X86::EAX;
5131 SDValue Chain = CurDAG->getCopyToReg(
Node->getOperand(0), dl, PtrReg,
5135 if (IntNo == Intrinsic::x86_sse3_monitor ||
5136 IntNo == Intrinsic::x86_monitorx) {
5138 Chain = CurDAG->getCopyToReg(Chain, dl, X86::ECX,
Node->getOperand(3),
5141 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EDX,
Node->getOperand(4),
5146 MachineSDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
5148 ReplaceNode(
Node, CNode);
5154 case Intrinsic::x86_tilestored64_internal: {
5155 unsigned Opc = X86::PTILESTOREDV;
5158 SDValue Scale = getI8Imm(1, dl);
5160 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5161 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5165 Node->getOperand(3),
5171 Node->getOperand(6),
5173 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5174 ReplaceNode(
Node, CNode);
5177 case Intrinsic::x86_tileloadd64:
5178 case Intrinsic::x86_tileloaddt164:
5179 case Intrinsic::x86_tilestored64: {
5180 if (!Subtarget->hasAMXTILE())
5185 case Intrinsic::x86_tileloadd64: Opc = X86::PTILELOADD;
break;
5186 case Intrinsic::x86_tileloaddt164: Opc = X86::PTILELOADDT1;
break;
5187 case Intrinsic::x86_tilestored64: Opc = X86::PTILESTORED;
break;
5190 unsigned TIndex =
Node->getConstantOperandVal(2);
5191 SDValue TReg = getI8Imm(TIndex, dl);
5193 SDValue Scale = getI8Imm(1, dl);
5195 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5196 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5199 if (Opc == X86::PTILESTORED) {
5201 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5204 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5206 ReplaceNode(
Node, CNode);
5214 if (Subtarget->isTargetNaCl())
5218 if (Subtarget->isTarget64BitILP32()) {
5223 assert(
Target.getValueType() == MVT::i32 &&
"Unexpected VT!");
5224 SDValue ZextTarget = CurDAG->getZExtOrTrunc(
Target, dl, MVT::i64);
5225 SDValue Brind = CurDAG->getNode(Opcode, dl, MVT::Other,
5226 Node->getOperand(0), ZextTarget);
5228 SelectCode(ZextTarget.
getNode());
5235 ReplaceNode(
Node, getGlobalBaseReg());
5243 CurDAG->RemoveDeadNode(
Node);
5249 if (matchBitExtract(
Node))
5254 if (tryShiftAmountMod(
Node))
5259 uint8_t
Imm =
Node->getConstantOperandVal(3);
5261 Node->getOperand(1),
Node->getOperand(2), Imm))
5267 if (tryVPTERNLOG(
Node))
5277 tryVPTESTM(
Node, N0, N1))
5280 tryVPTESTM(
Node, N1, N0))
5286 CurDAG->RemoveDeadNode(
Node);
5289 if (matchBitExtract(
Node))
5297 if (tryShrinkShlLogicImm(
Node))
5301 if (tryVPTERNLOG(
Node))
5316 if (!CurDAG->shouldOptForSize())
5320 if (NVT != MVT::i8 && NVT != MVT::i16 && NVT != MVT::i32 && NVT != MVT::i64)
5326 auto *Cst = dyn_cast<ConstantSDNode>(N1);
5330 int64_t Val = Cst->getSExtValue();
5334 if (!isInt<8>(Val) && !isInt<32>(Val))
5338 if (Opcode ==
ISD::ADD && (Val == 1 || Val == -1))
5342 if (!shouldAvoidImmediateInstFormsForSize(N1.
getNode()))
5346 unsigned ROpc, MOpc;
5455 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5456 if (tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5458 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5459 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5463 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N0)->getMemOperand()});
5465 CurDAG->RemoveDeadNode(
Node);
5470 CurDAG->SelectNodeTo(
Node, ROpc, NVT, MVT::i32, N0, N1);
5483 unsigned LoReg, ROpc, MOpc;
5488 ROpc = Opcode ==
X86ISD::SMUL ? X86::IMUL8r : X86::MUL8r;
5489 MOpc = Opcode ==
X86ISD::SMUL ? X86::IMUL8m : X86::MUL8m;
5508 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5509 bool FoldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5512 FoldedLoad = tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5517 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5526 VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5528 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32, MVT::Other);
5532 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5537 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5543 VTs = CurDAG->getVTList(NVT, MVT::i32);
5545 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32);
5547 CNode = CurDAG->getMachineNode(ROpc, dl, VTs, {N1, InGlue});
5552 CurDAG->RemoveDeadNode(
Node);
5562 unsigned LoReg, HiReg;
5564 bool UseMULX = !IsSigned && Subtarget->hasBMI2();
5569 Opc = UseMULXHi ? X86::MULX32Hrr
5571 : IsSigned ?
X86::IMUL32r
5573 MOpc = UseMULXHi ? X86::MULX32Hrm
5575 : IsSigned ?
X86::IMUL32m
5577 LoReg = UseMULX ? X86::EDX : X86::EAX;
5581 Opc = UseMULXHi ? X86::MULX64Hrr
5583 : IsSigned ?
X86::IMUL64r
5585 MOpc = UseMULXHi ? X86::MULX64Hrm
5587 : IsSigned ?
X86::IMUL64m
5589 LoReg = UseMULX ? X86::RDX : X86::RAX;
5594 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5595 bool foldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5598 foldedLoad = tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5603 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5612 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
5613 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5616 }
else if (UseMULX) {
5617 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other);
5618 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5623 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5624 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5630 ReplaceUses(N1.
getValue(1), Chain);
5632 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5634 SDValue Ops[] = { N1, InGlue };
5636 SDVTList VTs = CurDAG->getVTList(NVT);
5637 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5639 }
else if (UseMULX) {
5640 SDVTList VTs = CurDAG->getVTList(NVT, NVT);
5641 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5645 SDVTList VTs = CurDAG->getVTList(MVT::Glue);
5646 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5654 assert(LoReg &&
"Register for low half is not defined!");
5655 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg,
5666 assert(HiReg &&
"Register for high half is not defined!");
5667 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg,
5676 CurDAG->RemoveDeadNode(
Node);
5685 unsigned ROpc, MOpc;
5690 case MVT::i8: ROpc = X86::DIV8r; MOpc = X86::DIV8m;
break;
5691 case MVT::i16: ROpc = X86::DIV16r; MOpc = X86::DIV16m;
break;
5692 case MVT::i32: ROpc = X86::DIV32r; MOpc = X86::DIV32m;
break;
5693 case MVT::i64: ROpc = X86::DIV64r; MOpc = X86::DIV64m;
break;
5698 case MVT::i8: ROpc = X86::IDIV8r; MOpc = X86::IDIV8m;
break;
5699 case MVT::i16: ROpc = X86::IDIV16r; MOpc = X86::IDIV16m;
break;
5700 case MVT::i32: ROpc = X86::IDIV32r; MOpc = X86::IDIV32m;
break;
5701 case MVT::i64: ROpc = X86::IDIV64r; MOpc = X86::IDIV64m;
break;
5705 unsigned LoReg, HiReg, ClrReg;
5706 unsigned SExtOpcode;
5710 LoReg = X86::AL; ClrReg = HiReg = X86::AH;
5714 LoReg = X86::AX; HiReg = X86::DX;
5716 SExtOpcode = X86::CWD;
5719 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX;
5720 SExtOpcode = X86::CDQ;
5723 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX;
5724 SExtOpcode = X86::CQO;
5728 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5729 bool foldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5730 bool signBitIsZero = CurDAG->SignBitIsZero(N0);
5733 if (NVT == MVT::i8) {
5736 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain;
5738 if (tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5740 unsigned Opc = (
isSigned && !signBitIsZero) ? X86::MOVSX16rm8
5742 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, MVT::Other, Ops);
5744 ReplaceUses(N0.
getValue(1), Chain);
5746 CurDAG->setNodeMemRefs(Move, {cast<LoadSDNode>(N0)->getMemOperand()});
5748 unsigned Opc = (
isSigned && !signBitIsZero) ? X86::MOVSX16rr8
5750 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, N0);
5751 Chain = CurDAG->getEntryNode();
5753 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX,
SDValue(Move, 0),
5758 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
5759 LoReg, N0,
SDValue()).getValue(1);
5763 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InGlue),0);
5766 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
5768 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
5772 SDValue(CurDAG->getMachineNode(
5773 TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode,
5774 CurDAG->getTargetConstant(X86::sub_16bit, dl,
5782 SDValue(CurDAG->getMachineNode(
5783 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
5784 CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode,
5785 CurDAG->getTargetConstant(X86::sub_32bit, dl,
5793 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg,
5794 ClrNode, InGlue).getValue(1);
5802 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops);
5807 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5810 SDValue(CurDAG->getMachineNode(ROpc, dl, MVT::Glue, N1, InGlue), 0);
5820 if (HiReg == X86::AH && !
SDValue(
Node, 1).use_empty()) {
5821 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8);
5822 unsigned AHExtOpcode =
5823 isSigned ? X86::MOVSX32rr8_NOREX : X86::MOVZX32rr8_NOREX;
5825 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32,
5826 MVT::Glue, AHCopy, InGlue);
5831 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result);
5839 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5840 LoReg, NVT, InGlue);
5841 InGlue =
Result.getValue(2);
5848 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5849 HiReg, NVT, InGlue);
5850 InGlue =
Result.getValue(2);
5855 CurDAG->RemoveDeadNode(
Node);
5864 SDValue N0 =
Node->getOperand(IsStrictCmp ? 1 : 0);
5865 SDValue N1 =
Node->getOperand(IsStrictCmp ? 2 : 1);
5871 if (Subtarget->canUseCMOV())
5880 Opc = IsSignaling ? X86::COM_Fpr32 : X86::UCOM_Fpr32;
5883 Opc = IsSignaling ? X86::COM_Fpr64 : X86::UCOM_Fpr64;
5886 Opc = IsSignaling ? X86::COM_Fpr80 : X86::UCOM_Fpr80;
5891 IsStrictCmp ?
Node->getOperand(0) : CurDAG->getEntryNode();
5894 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5895 Chain =
SDValue(CurDAG->getMachineNode(Opc, dl, VTs, {N0, N1, Chain}), 0);
5898 Glue =
SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N0, N1), 0);
5903 SDValue(CurDAG->getMachineNode(X86::FNSTSW16r, dl, MVT::i16, Glue), 0);
5907 CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, MVT::i8, FNSTSW);
5911 assert(Subtarget->canUseLAHFSAHF() &&
5912 "Target doesn't support SAHF or FCOMI?");
5913 SDValue AH = CurDAG->getCopyToReg(Chain, dl, X86::AH, Extract,
SDValue());
5916 CurDAG->getMachineNode(X86::SAHF, dl, MVT::i32, AH.
getValue(1)), 0);
5922 CurDAG->RemoveDeadNode(
Node);
5943 unsigned TestOpc = CmpVT == MVT::i64 ? X86::TEST64rr
5946 NewNode = CurDAG->getMachineNode(TestOpc, dl, MVT::i32, BEXTR, BEXTR);
5948 CurDAG->RemoveDeadNode(
Node);
5962 auto *MaskC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
5980 unsigned TestOpcode;
5988 if (LeadingZeros == 0 && SavesBytes) {
5993 ShiftAmt = TrailingZeros;
5995 TestOpcode = X86::TEST64rr;
5996 }
else if (TrailingZeros == 0 && SavesBytes) {
6001 ShiftAmt = LeadingZeros;
6003 TestOpcode = X86::TEST64rr;
6004 }
else if (MaskC->hasOneUse() && !isInt<32>(Mask)) {
6007 unsigned PopCount = 64 - LeadingZeros - TrailingZeros;
6008 if (PopCount == 8) {
6010 ShiftAmt = TrailingZeros;
6011 SubRegIdx = X86::sub_8bit;
6013 TestOpcode = X86::TEST8rr;
6014 }
else if (PopCount == 16) {
6016 ShiftAmt = TrailingZeros;
6017 SubRegIdx = X86::sub_16bit;
6018 SubRegVT = MVT::i16;
6019 TestOpcode = X86::TEST16rr;
6020 }
else if (PopCount == 32) {
6022 ShiftAmt = TrailingZeros;
6023 SubRegIdx = X86::sub_32bit;
6024 SubRegVT = MVT::i32;
6025 TestOpcode = X86::TEST32rr;
6029 SDValue ShiftC = CurDAG->getTargetConstant(ShiftAmt, dl, MVT::i64);
6031 CurDAG->getMachineNode(ShiftOpcode, dl, MVT::i64, MVT::i32,
6034 if (SubRegIdx != 0) {
6036 CurDAG->getTargetExtractSubreg(SubRegIdx, dl, SubRegVT, Shift);
6039 CurDAG->getMachineNode(TestOpcode, dl, MVT::i32, Shift, Shift);
6047 unsigned ROpc, MOpc;
6054 if (isUInt<8>(Mask) &&
6055 (!(Mask & 0x80) || CmpVT == MVT::i8 ||
6059 SubRegOp = X86::sub_8bit;
6060 ROpc = X86::TEST8ri;
6061 MOpc = X86::TEST8mi;
6062 }
else if (OptForMinSize && isUInt<16>(Mask) &&
6063 (!(Mask & 0x8000) || CmpVT == MVT::i16 ||
6070 SubRegOp = X86::sub_16bit;
6071 ROpc = X86::TEST16ri;
6072 MOpc = X86::TEST16mi;
6073 }
else if (isUInt<32>(Mask) && N0.
getValueType() != MVT::i16 &&
6074 ((!(Mask & 0x80000000) &&
6077 (CmpVT != MVT::i16 || !(Mask & 0x8000))) ||
6078 CmpVT == MVT::i32 ||
6086 SubRegOp = X86::sub_32bit;
6087 ROpc = X86::TEST32ri;
6088 MOpc = X86::TEST32mi;
6094 SDValue Imm = CurDAG->getTargetConstant(Mask, dl, VT);
6099 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
6100 if (tryFoldLoad(
Node, N0.
getNode(), Reg, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
6102 if (!LoadN->isSimple()) {
6104 if ((MOpc == X86::TEST8mi && NumVolBits != 8) ||
6105 (MOpc == X86::TEST16mi && NumVolBits != 16) ||
6106 (MOpc == X86::TEST32mi && NumVolBits != 32))
6110 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
6111 Reg.getOperand(0) };
6112 NewNode = CurDAG->getMachineNode(MOpc, dl, MVT::i32, MVT::Other, Ops);
6114 ReplaceUses(
Reg.getValue(1),
SDValue(NewNode, 1));
6116 CurDAG->setNodeMemRefs(NewNode,
6117 {cast<LoadSDNode>(Reg)->getMemOperand()});
6121 Reg = CurDAG->getTargetExtractSubreg(SubRegOp, dl, VT, Reg);
6123 NewNode = CurDAG->getMachineNode(ROpc, dl, MVT::i32, Reg, Imm);
6126 ReplaceNode(
Node, NewNode);
6132 if (!Subtarget->hasSSE42())
6138 bool MayFoldLoad = !NeedIndex || !NeedMask;
6143 Subtarget->hasAVX() ? X86::VPCMPISTRMrri : X86::PCMPISTRMrri;
6145 Subtarget->hasAVX() ? X86::VPCMPISTRMrmi : X86::PCMPISTRMrmi;
6146 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8,
Node);
6149 if (NeedIndex || !NeedMask) {
6151 Subtarget->hasAVX() ? X86::VPCMPISTRIrri : X86::PCMPISTRIrri;
6153 Subtarget->hasAVX() ? X86::VPCMPISTRIrmi : X86::PCMPISTRIrmi;
6154 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32,
Node);
6160 CurDAG->RemoveDeadNode(
Node);
6164 if (!Subtarget->hasSSE42())
6168 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX,
6169 Node->getOperand(1),
6171 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EDX,
6172 Node->getOperand(3), InGlue).getValue(1);
6177 bool MayFoldLoad = !NeedIndex || !NeedMask;
6182 Subtarget->hasAVX() ? X86::VPCMPESTRMrri : X86::PCMPESTRMrri;
6184 Subtarget->hasAVX() ? X86::VPCMPESTRMrmi : X86::PCMPESTRMrmi;
6186 emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8,
Node, InGlue);
6189 if (NeedIndex || !NeedMask) {
6191 Subtarget->hasAVX() ? X86::VPCMPESTRIrri : X86::PCMPESTRIrri;
6193 Subtarget->hasAVX() ? X86::VPCMPESTRIrmi : X86::PCMPESTRIrmi;
6194 CNode = emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32,
Node, InGlue);
6199 CurDAG->RemoveDeadNode(
Node);
6211 if (foldLoadStoreIntoMemOperand(
Node))
6216 MVT VT =
Node->getSimpleValueType(0);
6218 if (Subtarget->hasSBBDepBreaking()) {
6223 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
6228 unsigned Opc = VT == MVT::i64 ? X86::SETB_C64r : X86::SETB_C32r;
6229 MVT SetVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
6231 CurDAG->getMachineNode(Opc, dl, SetVT, EFLAGS, EFLAGS.
getValue(1)),
6240 if (VT == MVT::i8 || VT == MVT::i16) {
6241 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6242 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6246 CurDAG->RemoveDeadNode(
Node);
6260 MVT VT =
Node->getSimpleValueType(0);
6261 if (VT == MVT::i8 || VT == MVT::i16) {
6262 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6263 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6268 CurDAG->RemoveDeadNode(
Node);
6274 auto *Mgt = cast<X86MaskedGatherSDNode>(
Node);
6275 SDValue IndexOp = Mgt->getIndex();
6278 MVT ValueVT =
Node->getSimpleValueType(0);
6279 MVT MaskVT =
Mask.getSimpleValueType();
6296 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6297 Opc = IsFP ? X86::VGATHERDPSZ128rm : X86::VPGATHERDDZ128rm;
6298 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6299 Opc = IsFP ? X86::VGATHERDPSZ256rm : X86::VPGATHERDDZ256rm;
6300 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6301 Opc = IsFP ? X86::VGATHERDPSZrm : X86::VPGATHERDDZrm;
6302 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6303 Opc = IsFP ? X86::VGATHERDPDZ128rm : X86::VPGATHERDQZ128rm;
6304 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6305 Opc = IsFP ? X86::VGATHERDPDZ256rm : X86::VPGATHERDQZ256rm;
6306 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6307 Opc = IsFP ? X86::VGATHERDPDZrm : X86::VPGATHERDQZrm;
6308 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6309 Opc = IsFP ? X86::VGATHERQPSZ128rm : X86::VPGATHERQDZ128rm;
6310 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6311 Opc = IsFP ? X86::VGATHERQPSZ256rm : X86::VPGATHERQDZ256rm;
6312 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6313 Opc = IsFP ? X86::VGATHERQPSZrm : X86::VPGATHERQDZrm;
6314 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6315 Opc = IsFP ? X86::VGATHERQPDZ128rm : X86::VPGATHERQQZ128rm;
6316 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6317 Opc = IsFP ? X86::VGATHERQPDZ256rm : X86::VPGATHERQQZ256rm;
6318 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6319 Opc = IsFP ? X86::VGATHERQPDZrm : X86::VPGATHERQQZrm;
6321 assert(
EVT(MaskVT) ==
EVT(ValueVT).changeVectorElementTypeToInteger() &&
6322 "Unexpected mask VT!");
6323 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6324 Opc = IsFP ? X86::VGATHERDPSrm : X86::VPGATHERDDrm;
6325 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6326 Opc = IsFP ? X86::VGATHERDPSYrm : X86::VPGATHERDDYrm;
6327 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6328 Opc = IsFP ? X86::VGATHERDPDrm : X86::VPGATHERDQrm;
6329 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6330 Opc = IsFP ? X86::VGATHERDPDYrm : X86::VPGATHERDQYrm;
6331 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6332 Opc = IsFP ? X86::VGATHERQPSrm : X86::VPGATHERQDrm;
6333 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6334 Opc = IsFP ? X86::VGATHERQPSYrm : X86::VPGATHERQDYrm;
6335 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6336 Opc = IsFP ? X86::VGATHERQPDrm : X86::VPGATHERQQrm;
6337 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6338 Opc = IsFP ? X86::VGATHERQPDYrm : X86::VPGATHERQQYrm;
6345 if (!selectVectorAddr(Mgt, Mgt->getBasePtr(), IndexOp, Mgt->getScale(),
6349 SDValue PassThru = Mgt->getPassThru();
6350 SDValue Chain = Mgt->getChain();
6352 SDVTList VTs = CurDAG->getVTList(ValueVT, MaskVT, MVT::Other);
6357 Index, Disp, Segment, Chain};
6358 NewNode = CurDAG->getMachineNode(Opc,
SDLoc(dl), VTs, Ops);
6361 Disp, Segment,
Mask, Chain};
6362 NewNode = CurDAG->getMachineNode(Opc,
SDLoc(dl), VTs, Ops);
6364 CurDAG->setNodeMemRefs(NewNode, {Mgt->getMemOperand()});
6367 CurDAG->RemoveDeadNode(
Node);
6371 auto *Sc = cast<X86MaskedScatterSDNode>(
Node);
6373 SDValue IndexOp = Sc->getIndex();
6375 MVT ValueVT =
Value.getSimpleValueType();
6390 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6391 Opc = IsFP ? X86::VSCATTERDPSZ128mr : X86::VPSCATTERDDZ128mr;
6392 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6393 Opc = IsFP ? X86::VSCATTERDPSZ256mr : X86::VPSCATTERDDZ256mr;
6394 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6395 Opc = IsFP ? X86::VSCATTERDPSZmr : X86::VPSCATTERDDZmr;
6396 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6397 Opc = IsFP ? X86::VSCATTERDPDZ128mr : X86::VPSCATTERDQZ128mr;
6398 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6399 Opc = IsFP ? X86::VSCATTERDPDZ256mr : X86::VPSCATTERDQZ256mr;
6400 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6401 Opc = IsFP ? X86::VSCATTERDPDZmr : X86::VPSCATTERDQZmr;
6402 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6403 Opc = IsFP ? X86::VSCATTERQPSZ128mr : X86::VPSCATTERQDZ128mr;
6404 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6405 Opc = IsFP ? X86::VSCATTERQPSZ256mr : X86::VPSCATTERQDZ256mr;
6406 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6407 Opc = IsFP ? X86::VSCATTERQPSZmr : X86::VPSCATTERQDZmr;
6408 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6409 Opc = IsFP ? X86::VSCATTERQPDZ128mr : X86::VPSCATTERQQZ128mr;
6410 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6411 Opc = IsFP ? X86::VSCATTERQPDZ256mr : X86::VPSCATTERQQZ256mr;
6412 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6413 Opc = IsFP ? X86::VSCATTERQPDZmr : X86::VPSCATTERQQZmr;
6418 if (!selectVectorAddr(Sc, Sc->getBasePtr(), IndexOp, Sc->getScale(),
6423 SDValue Chain = Sc->getChain();
6425 SDVTList VTs = CurDAG->getVTList(
Mask.getValueType(), MVT::Other);
6429 CurDAG->setNodeMemRefs(NewNode, {Sc->getMemOperand()});
6431 CurDAG->RemoveDeadNode(
Node);
6437 cast<SrcValueSDNode>(
Node->getOperand(1))->getValue());
6439 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6441 TargetOpcode::PREALLOCATED_SETUP, dl, MVT::Other, CallIdValue, Chain);
6443 CurDAG->RemoveDeadNode(
Node);
6449 cast<SrcValueSDNode>(
Node->getOperand(1))->getValue());
6451 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6454 Ops[0] = CallIdValue;
6458 TargetOpcode::PREALLOCATED_ARG, dl,
6459 CurDAG->getVTList(TLI->
getPointerTy(CurDAG->getDataLayout()),
6464 CurDAG->RemoveDeadNode(
Node);
6471 if (!Subtarget->hasWIDEKL())
6475 switch (
Node->getOpcode()) {
6490#undef GET_EGPR_IF_ENABLED
6500 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(2),
6502 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(3),
6504 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM2,
Node->getOperand(4),
6506 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM3,
Node->getOperand(5),
6508 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM4,
Node->getOperand(6),
6510 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM5,
Node->getOperand(7),
6512 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM6,
Node->getOperand(8),
6514 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM7,
Node->getOperand(9),
6518 Opcode, dl,
Node->getVTList(),
6519 {Base, Scale, Index, Disp, Segment, Chain, Chain.getValue(1)});
6520 CurDAG->setNodeMemRefs(Res, cast<MemSDNode>(
Node)->getMemOperand());
6521 ReplaceNode(
Node, Res);
6529bool X86DAGToDAGISel::SelectInlineAsmMemoryOperand(
6531 std::vector<SDValue> &OutOps) {
6532 SDValue Op0, Op1, Op2, Op3, Op4;
6533 switch (ConstraintID) {
6536 case InlineAsm::ConstraintCode::o:
6537 case InlineAsm::ConstraintCode::v:
6538 case InlineAsm::ConstraintCode::m:
6539 case InlineAsm::ConstraintCode::X:
6540 case InlineAsm::ConstraintCode::p:
6541 if (!selectAddr(
nullptr,
Op, Op0, Op1, Op2, Op3, Op4))
6546 OutOps.push_back(Op0);
6547 OutOps.push_back(Op1);
6548 OutOps.push_back(Op2);
6549 OutOps.push_back(Op3);
6550 OutOps.push_back(Op4);
6558 return new X86DAGToDAGISel(
TM, OptLevel);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool isRIPRelative(const MCInst &MI, const MCInstrInfo &MCII)
Check if the instruction uses RIP relative addressing.
static bool isLegalMaskCompare(SDNode *N, const X86Subtarget *Subtarget)
static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, unsigned LoadOpNo, LoadSDNode *&LoadNode, SDValue &InputChain)
Check whether or not the chain ending in StoreNode is suitable for doing the {load; op; store} to mod...
#define GET_EGPR_IF_ENABLED(OPC)
static bool needBWI(MVT VT)
static unsigned getVPTESTMOpc(MVT TestVT, bool IsTestN, bool FoldedLoad, bool FoldedBCast, bool Masked)
static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM, const X86Subtarget &Subtarget)
static bool mayUseCarryFlag(X86::CondCode CC)
static cl::opt< bool > EnablePromoteAnyextLoad("x86-promote-anyext-load", cl::init(true), cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden)
cl::opt< bool > IndirectBranchTracking
static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, SDValue Call, SDValue OrigChain)
Replace the original chain operand of the call with load's chain operand and move load below the call...
static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N)
#define GET_ND_IF_ENABLED(OPC)
#define VPTESTM_BROADCAST_CASES(SUFFIX)
static cl::opt< bool > AndImmShrink("x86-and-imm-shrink", cl::init(true), cl::desc("Enable setting constant bits to reduce size of mask immediates"), cl::Hidden)
static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, X86ISelAddressMode &AM)
#define VPTESTM_FULL_CASES(SUFFIX)
static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq)
Return true if call address is a load and it can be moved below CALLSEQ_START and the chains leading ...
static bool isDispSafeForFrameIndex(int64_t Val)
static bool isEndbrImm64(uint64_t Imm)
#define GET_ND_IF_ENABLED(OPC)
DEMANGLE_DUMP_METHOD void dump() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
APInt trunc(unsigned width) const
Truncate to new width.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countl_zero() const
The APInt version of std::countl_zero.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
bool isOne() const
Determine if this is a value of 1.
unsigned countr_one() const
Count the number of trailing one bits.
The address of a basic block.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
std::optional< ConstantRange > getAbsoluteSymbolRange() const
If this is an absolute symbol reference, returns the range of the symbol, otherwise returns std::null...
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
Describe properties that are true of each instruction in the target description file.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
bool is128BitVector() const
Return true if this is a 128-bit vector type.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool is512BitVector() const
Return true if this is a 512-bit vector type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const Module * getModule() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isNonTemporal() const
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
static int getUninvalidatedNodeId(SDNode *N)
virtual void emitFunctionEntryCode()
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
static constexpr unsigned MaxRecursionDepth
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
static Type * getVoidTy(LLVMContext &C)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void dump() const
Support for debugging, callable in GDB: V->dump()
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
size_t getPreallocatedIdForCallSite(const Value *CS)
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BRIND
BRIND - Indirect branch.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
@ GlobalBaseReg
The result of the mflr at function entry, used for PIC code.
@ X86
Windows x64, Windows Itanium (IA-64)
Reg
All possible values of the reg field in the ModR/M byte.
@ MO_NO_FLAG
MO_NO_FLAG - No flag for the operand.
@ EVEX
EVEX - Specifies that this instruction use EVEX form which provides syntax support up to 32 512-bit r...
@ VEX
VEX - encoding using 0xC4/0xC5.
@ XOP
XOP - Opcode prefix used by XOP instructions.
@ FST
This instruction implements a truncating store from FP stack slots.
@ CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ CMP
X86 compare and logical compare instructions.
@ BLENDV
Dynamic (non-constant condition) vector blend where only the sign bits of the condition elements are ...
@ STRICT_FCMP
X86 strict FP compare instructions.
@ STRICT_CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ NT_BRIND
BRIND node with NoTrack prefix.
@ FSETCCM
X86 FP SETCC, similar to above, but with output as an i1 mask and and a version with SAE.
@ FXOR
Bitwise logical XOR of floating point values.
@ BRCOND
X86 conditional branches.
@ CALL
These operations represent an abstract X86 call instruction, which includes a bunch of information.
@ FANDN
Bitwise logical ANDNOT of floating point values.
@ GlobalBaseReg
On Darwin, this node represents the result of the popl at function entry, used for PIC code.
@ FLD
This instruction implements an extending load to FP stack slots.
@ TC_RETURN
Tail call return.
@ FOR
Bitwise logical OR of floating point values.
@ Wrapper
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress,...
@ ANDNP
Bitwise Logical AND NOT of Packed FP values.
@ FAND
Bitwise logical AND of floating point values.
@ CMOV
X86 conditional moves.
@ WrapperRIP
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
int getCondSrcNoFromDesc(const MCInstrDesc &MCID)
Return the source operand # for condition code by MCID.
bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into some other x86 instruction as a memory oper...
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement)
Returns true of the given offset can be fit into displacement field of the instruction.
bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs)
If Op is a constant whose elements are all the same constant or undefined, return true and return the...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
int popcount(T Value) noexcept
Count the number of set bits in a value.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
unsigned M1(unsigned Val)
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
FunctionPass * createX86ISelDag(X86TargetMachine &TM, CodeGenOptLevel OptLevel)
This pass converts a legalized DAG into a X86-specific DAG, ready for instruction scheduling.
CodeGenOptLevel
Code generation optimization level.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.