22#include "llvm/Config/llvm-config.h"
27#include "llvm/IR/IntrinsicsX86.h"
37#define DEBUG_TYPE "x86-isel"
38#define PASS_NAME "X86 DAG->DAG Instruction Selection"
40STATISTIC(NumLoadMoved,
"Number of loads moved below TokenFactor");
43 cl::desc(
"Enable setting constant bits to reduce size of mask immediates"),
47 "x86-promote-anyext-load",
cl::init(
true),
59 struct X86ISelAddressMode {
67 int Base_FrameIndex = 0;
76 const char *ES =
nullptr;
81 bool NegateIndex =
false;
83 X86ISelAddressMode() =
default;
85 bool hasSymbolicDisplacement()
const {
86 return GV !=
nullptr ||
CP !=
nullptr || ES !=
nullptr ||
87 MCSym !=
nullptr ||
JT != -1 || BlockAddr !=
nullptr;
90 bool hasBaseOrIndexReg()
const {
97 if (
BaseType != RegBase)
return false;
99 dyn_cast_or_null<RegisterSDNode>(Base_Reg.
getNode()))
100 return RegNode->getReg() == X86::RIP;
109#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
111 dbgs() <<
"X86ISelAddressMode " <<
this <<
'\n';
112 dbgs() <<
"Base_Reg ";
118 dbgs() <<
" Base.FrameIndex " << Base_FrameIndex <<
'\n';
119 dbgs() <<
" Scale " << Scale <<
'\n'
127 dbgs() <<
" Disp " << Disp <<
'\n'
149 dbgs() <<
" JT" <<
JT <<
" Align" << Alignment.
value() <<
'\n';
169 bool IndirectTlsSegRefs;
174 X86DAGToDAGISel() =
delete;
178 OptForMinSize(
false), IndirectTlsSegRefs(
false) {}
184 "indirect-tls-seg-refs");
189 "OptForMinSize implies OptForSize");
203#include "X86GenDAGISel.inc"
208 bool foldOffsetIntoAddress(
uint64_t Offset, X86ISelAddressMode &AM);
209 bool matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
210 bool AllowSegmentRegForX32 =
false);
211 bool matchWrapper(
SDValue N, X86ISelAddressMode &AM);
212 bool matchAddress(
SDValue N, X86ISelAddressMode &AM);
213 bool matchVectorAddress(
SDValue N, X86ISelAddressMode &AM);
214 bool matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
unsigned Depth);
217 bool matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
219 bool matchVectorAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
221 bool matchAddressBase(
SDValue N, X86ISelAddressMode &AM);
250 return tryFoldLoad(
P,
P,
N,
Base, Scale,
Index, Disp, Segment);
258 bool isProfitableToFormMaskedOp(
SDNode *
N)
const;
263 std::vector<SDValue> &OutOps)
override;
265 void emitSpecialCodeForMain();
267 inline void getAddressOperands(X86ISelAddressMode &AM,
const SDLoc &
DL,
271 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
272 Base = CurDAG->getTargetFrameIndex(
273 AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout()));
274 else if (AM.Base_Reg.getNode())
277 Base = CurDAG->getRegister(0, VT);
279 Scale = getI8Imm(AM.Scale,
DL);
281#define GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC)
283 if (AM.NegateIndex) {
291 if (AM.IndexReg.getNode())
294 Index = CurDAG->getRegister(0, VT);
299 Disp = CurDAG->getTargetGlobalAddress(AM.GV,
SDLoc(),
303 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment,
304 AM.Disp, AM.SymbolFlags);
306 assert(!AM.Disp &&
"Non-zero displacement is ignored with ES.");
307 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
308 }
else if (AM.MCSym) {
309 assert(!AM.Disp &&
"Non-zero displacement is ignored with MCSym.");
310 assert(AM.SymbolFlags == 0 &&
"oo");
311 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32);
312 }
else if (AM.JT != -1) {
313 assert(!AM.Disp &&
"Non-zero displacement is ignored with JT.");
314 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
315 }
else if (AM.BlockAddr)
316 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
319 Disp = CurDAG->getTargetConstant(AM.Disp,
DL, MVT::i32);
321 if (AM.Segment.getNode())
322 Segment = AM.Segment;
324 Segment = CurDAG->getRegister(0, MVT::i16);
333 bool shouldAvoidImmediateInstFormsForSize(
SDNode *
N)
const {
339 if (!CurDAG->shouldOptForSize())
349 if (
User->isMachineOpcode()) {
372 auto *
C = dyn_cast<ConstantSDNode>(
N);
373 if (
C && isInt<8>(
C->getSExtValue()))
393 (RegNode = dyn_cast_or_null<RegisterSDNode>(
395 if ((RegNode->
getReg() == X86::ESP) ||
396 (RegNode->
getReg() == X86::RSP))
405 return (UseCount > 1);
410 return CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
415 return CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
420 return CurDAG->getTargetConstant(Imm,
DL, MVT::i64);
425 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
427 MVT VecVT =
N->getOperand(0).getSimpleValueType();
433 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
435 MVT VecVT =
N->getSimpleValueType(0);
439 SDValue getPermuteVINSERTCommutedImmediate(
SDNode *
N,
unsigned VecWidth,
441 assert(VecWidth == 128 &&
"Unexpected vector width");
443 MVT VecVT =
N->getSimpleValueType(0);
445 assert((InsertIdx == 0 || InsertIdx == 1) &&
"Bad insertf128 index");
448 return getI8Imm(InsertIdx ? 0x02 : 0x30,
DL);
453 MVT VT =
N->getSimpleValueType(0);
456 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
458 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
459 if (VT == MVT::i64) {
461 CurDAG->getMachineNode(
462 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
463 CurDAG->getTargetConstant(0, dl, MVT::i64), Zero,
464 CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)),
469 unsigned Opcode =
N->getOpcode();
471 "Unexpected opcode for SBB materialization");
472 unsigned FlagOpIndex = Opcode ==
X86ISD::SBB ? 2 : 1;
474 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
475 N->getOperand(FlagOpIndex),
SDValue());
479 unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr;
480 MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
481 VTs = CurDAG->getVTList(SBBVT, MVT::i32);
483 CurDAG->getMachineNode(Opc, dl, VTs,
484 {Zero, Zero, EFLAGS, EFLAGS.getValue(1)}),
490 bool isUnneededShiftMask(
SDNode *
N,
unsigned Width)
const {
492 const APInt &Val =
N->getConstantOperandAPInt(1);
497 APInt Mask = Val | CurDAG->computeKnownBits(
N->getOperand(0)).Zero;
498 return Mask.countr_one() >= Width;
504 SDNode *getGlobalBaseReg();
515 return Subtarget->getInstrInfo();
528 bool isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const;
532 if (!
N->isNonTemporal())
535 unsigned StoreSize =
N->getMemoryVT().getStoreSize();
537 if (
N->getAlign().value() < StoreSize)
546 return Subtarget->hasSSE41();
548 return Subtarget->hasAVX2();
550 return Subtarget->hasAVX512();
554 bool foldLoadStoreIntoMemOperand(
SDNode *
Node);
557 bool shrinkAndImmediate(
SDNode *
N);
558 bool isMaskZeroExtended(
SDNode *
N)
const;
559 bool tryShiftAmountMod(
SDNode *
N);
560 bool tryShrinkShlLogicImm(
SDNode *
N);
566 bool tryMatchBitSelect(
SDNode *
N);
568 MachineSDNode *emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
570 MachineSDNode *emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
574 bool tryOptimizeRem8Extend(
SDNode *
N);
576 bool onlyUsesZeroFlag(
SDValue Flags)
const;
577 bool hasNoSignFlagUses(
SDValue Flags)
const;
578 bool hasNoCarryFlagUses(
SDValue Flags)
const;
582char X86DAGToDAGISel::ID = 0;
589 unsigned Opcode =
N->getOpcode();
596 EVT OpVT =
N->getOperand(0).getValueType();
600 OpVT =
N->getOperand(1).getValueType();
602 return Subtarget->hasVLX();
616bool X86DAGToDAGISel::isMaskZeroExtended(
SDNode *
N)
const {
629 if (OptLevel == CodeGenOptLevel::None)
639 if (useNonTemporalLoad(cast<LoadSDNode>(
N)))
644 switch (
U->getOpcode()) {
670 if (
auto *Imm = dyn_cast<ConstantSDNode>(Op1)) {
671 if (
Imm->getAPIntValue().isSignedIntN(8))
680 Imm->getAPIntValue().getBitWidth() == 64 &&
681 Imm->getAPIntValue().isIntN(32))
688 (
Imm->getAPIntValue() == UINT8_MAX ||
689 Imm->getAPIntValue() == UINT16_MAX ||
690 Imm->getAPIntValue() == UINT32_MAX))
696 (-
Imm->getAPIntValue()).isSignedIntN(8))
700 (-
Imm->getAPIntValue()).isSignedIntN(8) &&
701 hasNoCarryFlagUses(
SDValue(U, 1)))
726 if (
U->getOperand(0).getOpcode() ==
ISD::SHL &&
730 if (
U->getOperand(1).getOpcode() ==
ISD::SHL &&
738 auto *
C = dyn_cast<ConstantSDNode>(U0.
getOperand(0));
739 if (
C &&
C->getSExtValue() == -2)
744 auto *
C = dyn_cast<ConstantSDNode>(U1.
getOperand(0));
745 if (
C &&
C->getSExtValue() == -2)
759 if (isa<ConstantSDNode>(
U->getOperand(1)))
780bool X86DAGToDAGISel::isProfitableToFormMaskedOp(
SDNode *
N)
const {
783 "Unexpected opcode!");
788 return N->getOperand(1).hasOneUse();
797 if (Chain.
getNode() == Load.getNode())
801 "Unexpected chain operand");
815 Load.getOperand(1), Load.getOperand(2));
819 Ops.
append(Call->op_begin() + 1, Call->op_end());
833 if (Callee.getNode() == Chain.
getNode() || !Callee.hasOneUse())
835 auto *LD = dyn_cast<LoadSDNode>(Callee.getNode());
853 if (isa<MemSDNode>(Chain.
getNode()) &&
854 cast<MemSDNode>(Chain.
getNode())->writeMem())
860 Callee.getValue(1).hasOneUse())
868 if ((Imm & 0x00FFFFFF) != 0x0F1EFA)
871 uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64,
872 0x65, 0x66, 0x67, 0xf0, 0xf2};
875 uint8_t Byte = (Imm >> i) & 0xFF;
887 return (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v64i8);
890void X86DAGToDAGISel::PreprocessISelDAG() {
891 bool MadeChange =
false;
893 E = CurDAG->allnodes_end();
I != E; ) {
912 MVT VT =
N->getSimpleValueType(0);
913 int64_t
Imm = cast<ConstantSDNode>(
N)->getSExtValue();
914 int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB;
921 SDValue Complement = CurDAG->getConstant(~Imm, dl, VT,
false,
true);
922 Complement = CurDAG->getNOT(dl, Complement, VT);
924 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Complement);
934 if (
N->getOpcode() ==
X86ISD::AND && !
N->hasAnyUseOfValue(1)) {
936 N->getOperand(0),
N->getOperand(1));
938 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
962 auto mayPreventLoadFold = [&]() {
964 N->getOpcode() ==
ISD::ADD && Subtarget->hasAVX() &&
965 !
N->getOperand(1).hasOneUse();
968 N->getSimpleValueType(0).isVector() && !mayPreventLoadFold()) {
974 MVT VT =
N->getSimpleValueType(0);
982 CurDAG->getNode(NewOpcode,
DL, VT,
N->getOperand(0),
AllOnes);
984 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
991 switch (
N->getOpcode()) {
993 MVT VT =
N->getSimpleValueType(0);
995 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1002 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1005 CurDAG->getIntPtrConstant(
Index, dl));
1008 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1017 MVT VT =
N->getSimpleValueType(0);
1019 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1021 auto *MemNode = cast<MemSDNode>(
N);
1023 SDVTList VTs = CurDAG->getVTList(NarrowVT, MVT::Other);
1024 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()};
1025 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode(
1027 MemNode->getMemOperand());
1030 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1033 CurDAG->getIntPtrConstant(
Index, dl));
1037 CurDAG->ReplaceAllUsesWith(
N, To);
1048 auto *Ld = cast<LoadSDNode>(
N);
1049 MVT VT =
N->getSimpleValueType(0);
1057 SDValue Chain = Ld->getChain();
1059 auto *UserLd = dyn_cast<LoadSDNode>(
User);
1060 MVT UserVT =
User->getSimpleValueType(0);
1062 UserLd->getBasePtr() ==
Ptr && UserLd->getChain() == Chain &&
1063 !
User->hasAnyUseOfValue(1) &&
1077 CurDAG->getIntPtrConstant(0, dl));
1078 SDValue Res = CurDAG->getBitcast(VT, Extract);
1082 CurDAG->ReplaceAllUsesWith(
N, To);
1091 EVT EleVT =
N->getOperand(0).getValueType().getVectorElementType();
1092 if (EleVT == MVT::i1)
1095 assert(Subtarget->hasSSE41() &&
"Expected SSE4.1 support!");
1096 assert(
N->getValueType(0).getVectorElementType() != MVT::i16 &&
1097 "We can't replace VSELECT with BLENDV in vXi16!");
1099 if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(
N->getOperand(0)) ==
1102 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
1103 CurDAG->getTargetConstant(0xCA,
SDLoc(
N), MVT::i8));
1106 N->getOperand(0),
N->getOperand(1),
1110 CurDAG->ReplaceAllUsesWith(
N,
R.getNode());
1123 if (!
N->getSimpleValueType(0).isVector())
1127 switch (
N->getOpcode()) {
1137 if (
N->isStrictFPOpcode())
1139 CurDAG->getNode(NewOpc,
SDLoc(
N), {
N->getValueType(0), MVT::Other},
1140 {
N->getOperand(0),
N->getOperand(1)});
1143 CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1146 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1156 if (!
N->getValueType(0).isVector())
1160 switch (
N->getOpcode()) {
1166 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1167 N->getOperand(0),
N->getOperand(1));
1169 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1178 if (!
N->getValueType(0).isVector())
1182 if (
N->getOperand(0).getScalarValueSizeInBits() == 1) {
1184 "Unexpected opcode for mask vector!");
1192 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1195 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1215 switch (
N->getOpcode()) {
1231 bool IsStrict =
N->isStrictFPOpcode();
1235 {
N->getValueType(0), MVT::Other},
1236 {
N->getOperand(0),
N->getOperand(1),
1237 CurDAG->getTargetConstant(Imm, dl, MVT::i32)});
1241 CurDAG->getTargetConstant(Imm, dl, MVT::i32));
1243 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1254 MVT VT =
N->getSimpleValueType(0);
1255 if (VT.
isVector() || VT == MVT::f128)
1258 MVT VecVT = VT == MVT::f64 ? MVT::v2f64
1259 : VT == MVT::f32 ? MVT::v4f32
1269 if (Subtarget->hasSSE2()) {
1274 switch (
N->getOpcode()) {
1281 Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1);
1284 Res = CurDAG->getNode(
N->getOpcode(), dl, VecVT, Op0, Op1);
1287 CurDAG->getIntPtrConstant(0, dl));
1289 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1296 if (OptLevel != CodeGenOptLevel::None &&
1299 !Subtarget->useIndirectThunkCalls() &&
1300 ((
N->getOpcode() ==
X86ISD::CALL && !Subtarget->slowTwoMemOps()) ||
1302 (Subtarget->is64Bit() ||
1303 !getTargetMachine().isPositionIndependent())))) {
1342 switch (
N->getOpcode()) {
1347 MVT SrcVT =
N->getOperand(0).getSimpleValueType();
1348 MVT DstVT =
N->getSimpleValueType(0);
1360 if (SrcIsSSE && DstIsSSE)
1363 if (!SrcIsSSE && !DstIsSSE) {
1368 if (
N->getConstantOperandVal(1))
1376 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1377 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1385 CurDAG->getEntryNode(), dl,
N->getOperand(0), MemTmp, MPI, MemVT);
1387 MemTmp, MPI, MemVT);
1394 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Result);
1403 MVT SrcVT =
N->getOperand(1).getSimpleValueType();
1404 MVT DstVT =
N->getSimpleValueType(0);
1416 if (SrcIsSSE && DstIsSSE)
1419 if (!SrcIsSSE && !DstIsSSE) {
1424 if (
N->getConstantOperandVal(2))
1432 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1433 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1443 SDVTList VTs = CurDAG->getVTList(MVT::Other);
1444 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), MemTmp};
1448 if (
N->getFlags().hasNoFPExcept()) {
1450 Flags.setNoFPExcept(
true);
1451 Store->setFlags(Flags);
1454 assert(SrcVT == MemVT &&
"Unexpected VT!");
1455 Store = CurDAG->getStore(
N->getOperand(0), dl,
N->getOperand(1), MemTmp,
1460 SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other);
1462 Result = CurDAG->getMemIntrinsicNode(
1465 if (
N->getFlags().hasNoFPExcept()) {
1467 Flags.setNoFPExcept(
true);
1471 assert(DstVT == MemVT &&
"Unexpected VT!");
1472 Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI);
1480 CurDAG->ReplaceAllUsesWith(
N,
Result.getNode());
1494 CurDAG->RemoveDeadNodes();
1498bool X86DAGToDAGISel::tryOptimizeRem8Extend(
SDNode *
N) {
1499 unsigned Opc =
N->getMachineOpcode();
1500 if (Opc != X86::MOVZX32rr8 && Opc != X86::MOVSX32rr8 &&
1501 Opc != X86::MOVSX64rr8)
1513 unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX
1514 : X86::MOVSX32rr8_NOREX;
1519 if (Opc == X86::MOVSX64rr8) {
1524 ReplaceUses(
N, Extend);
1533void X86DAGToDAGISel::PostprocessISelDAG() {
1535 if (
TM.getOptLevel() == CodeGenOptLevel::None)
1540 bool MadeChange =
false;
1541 while (Position != CurDAG->allnodes_begin()) {
1544 if (
N->use_empty() || !
N->isMachineOpcode())
1547 if (tryOptimizeRem8Extend(
N)) {
1554 unsigned Opc =
N->getMachineOpcode();
1555 if ((Opc == X86::TEST8rr || Opc == X86::TEST16rr ||
1556 Opc == X86::TEST32rr || Opc == X86::TEST64rr) &&
1557 N->getOperand(0) ==
N->getOperand(1) &&
1558 N->getOperand(0)->hasNUsesOfValue(2,
N->getOperand(0).getResNo()) &&
1559 N->getOperand(0).isMachineOpcode()) {
1561 unsigned N0Opc =
And.getMachineOpcode();
1562 if ((N0Opc == X86::AND8rr || N0Opc == X86::AND16rr ||
1563 N0Opc == X86::AND32rr || N0Opc == X86::AND64rr) &&
1564 !
And->hasAnyUseOfValue(1)) {
1569 ReplaceUses(
N,
Test);
1573 if ((N0Opc == X86::AND8rm || N0Opc == X86::AND16rm ||
1574 N0Opc == X86::AND32rm || N0Opc == X86::AND64rm) &&
1575 !
And->hasAnyUseOfValue(1)) {
1578 case X86::AND8rm: NewOpc = X86::TEST8mr;
break;
1579 case X86::AND16rm: NewOpc = X86::TEST16mr;
break;
1580 case X86::AND32rm: NewOpc = X86::TEST32mr;
break;
1581 case X86::AND64rm: NewOpc = X86::TEST64mr;
break;
1591 And.getOperand(6) };
1593 MVT::i32, MVT::Other, Ops);
1594 CurDAG->setNodeMemRefs(
1595 Test, cast<MachineSDNode>(
And.getNode())->memoperands());
1607 if ((Opc == X86::KORTESTBrr || Opc == X86::KORTESTWrr ||
1608 Opc == X86::KORTESTDrr || Opc == X86::KORTESTQrr) &&
1609 N->getOperand(0) ==
N->getOperand(1) &&
1610 N->isOnlyUserOf(
N->getOperand(0).getNode()) &&
1611 N->getOperand(0).isMachineOpcode() &&
1614 unsigned N0Opc =
And.getMachineOpcode();
1617 if (N0Opc == X86::KANDBrr ||
1618 (N0Opc == X86::KANDWrr && Subtarget->hasDQI()) ||
1619 N0Opc == X86::KANDDrr || N0Opc == X86::KANDQrr) {
1623 case X86::KORTESTBrr: NewOpc = X86::KTESTBrr;
break;
1624 case X86::KORTESTWrr: NewOpc = X86::KTESTWrr;
break;
1625 case X86::KORTESTDrr: NewOpc = X86::KTESTDrr;
break;
1626 case X86::KORTESTQrr: NewOpc = X86::KTESTQrr;
break;
1632 ReplaceUses(
N, KTest);
1639 if (Opc != TargetOpcode::SUBREG_TO_REG)
1642 unsigned SubRegIdx =
N->getConstantOperandVal(2);
1643 if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm)
1654 case X86::VMOVAPDrr:
case X86::VMOVUPDrr:
1655 case X86::VMOVAPSrr:
case X86::VMOVUPSrr:
1656 case X86::VMOVDQArr:
case X86::VMOVDQUrr:
1657 case X86::VMOVAPDYrr:
case X86::VMOVUPDYrr:
1658 case X86::VMOVAPSYrr:
case X86::VMOVUPSYrr:
1659 case X86::VMOVDQAYrr:
case X86::VMOVDQUYrr:
1660 case X86::VMOVAPDZ128rr:
case X86::VMOVUPDZ128rr:
1661 case X86::VMOVAPSZ128rr:
case X86::VMOVUPSZ128rr:
1662 case X86::VMOVDQA32Z128rr:
case X86::VMOVDQU32Z128rr:
1663 case X86::VMOVDQA64Z128rr:
case X86::VMOVDQU64Z128rr:
1664 case X86::VMOVAPDZ256rr:
case X86::VMOVUPDZ256rr:
1665 case X86::VMOVAPSZ256rr:
case X86::VMOVUPSZ256rr:
1666 case X86::VMOVDQA32Z256rr:
case X86::VMOVDQU32Z256rr:
1667 case X86::VMOVDQA64Z256rr:
case X86::VMOVDQU64Z256rr:
1672 if (!
In.isMachineOpcode() ||
1673 In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END)
1678 uint64_t TSFlags = getInstrInfo()->get(
In.getMachineOpcode()).TSFlags;
1686 CurDAG->UpdateNodeOperands(
N,
N->getOperand(0), In,
N->getOperand(2));
1691 CurDAG->RemoveDeadNodes();
1696void X86DAGToDAGISel::emitSpecialCodeForMain() {
1697 if (Subtarget->isTargetCygMing()) {
1699 auto &
DL = CurDAG->getDataLayout();
1702 CLI.setChain(CurDAG->getRoot())
1704 CurDAG->getExternalSymbol(
"__main", TLI->getPointerTy(
DL)),
1708 CurDAG->setRoot(
Result.second);
1712void X86DAGToDAGISel::emitFunctionEntryCode() {
1715 if (
F.hasExternalLinkage() &&
F.getName() ==
"main")
1716 emitSpecialCodeForMain();
1726 return isInt<31>(Val);
1730 X86ISelAddressMode &AM) {
1735 int64_t Val = AM.Disp +
Offset;
1738 if (Val != 0 && (AM.ES || AM.MCSym))
1742 if (Subtarget->is64Bit()) {
1745 AM.hasSymbolicDisplacement()))
1749 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
1768 if (Subtarget->isTarget64BitILP32() && !isUInt<31>(Val) &&
1769 !AM.hasBaseOrIndexReg())
1776bool X86DAGToDAGISel::matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
1777 bool AllowSegmentRegForX32) {
1789 if (
isNullConstant(Address) && AM.Segment.getNode() ==
nullptr &&
1790 !IndirectTlsSegRefs &&
1791 (Subtarget->isTargetGlibc() || Subtarget->isTargetAndroid() ||
1792 Subtarget->isTargetFuchsia())) {
1793 if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32)
1795 switch (
N->getPointerInfo().getAddrSpace()) {
1797 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
1800 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
1813bool X86DAGToDAGISel::matchWrapper(
SDValue N, X86ISelAddressMode &AM) {
1816 if (AM.hasSymbolicDisplacement())
1819 bool IsRIPRelTLS =
false;
1837 if (IsRIPRel && AM.hasBaseOrIndexReg())
1841 X86ISelAddressMode Backup = AM;
1845 if (
auto *
G = dyn_cast<GlobalAddressSDNode>(N0)) {
1846 AM.GV =
G->getGlobal();
1847 AM.SymbolFlags =
G->getTargetFlags();
1849 }
else if (
auto *CP = dyn_cast<ConstantPoolSDNode>(N0)) {
1850 AM.CP =
CP->getConstVal();
1851 AM.Alignment =
CP->getAlign();
1852 AM.SymbolFlags =
CP->getTargetFlags();
1854 }
else if (
auto *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
1855 AM.ES = S->getSymbol();
1856 AM.SymbolFlags = S->getTargetFlags();
1857 }
else if (
auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
1858 AM.MCSym = S->getMCSymbol();
1859 }
else if (
auto *J = dyn_cast<JumpTableSDNode>(N0)) {
1860 AM.JT = J->getIndex();
1861 AM.SymbolFlags = J->getTargetFlags();
1862 }
else if (
auto *BA = dyn_cast<BlockAddressSDNode>(N0)) {
1863 AM.BlockAddr = BA->getBlockAddress();
1864 AM.SymbolFlags = BA->getTargetFlags();
1865 Offset = BA->getOffset();
1870 if (Subtarget->is64Bit() && !IsRIPRel && AM.GV &&
1871 TM.isLargeGlobalValue(AM.GV)) {
1876 if (foldOffsetIntoAddress(
Offset, AM)) {
1882 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64));
1890bool X86DAGToDAGISel::matchAddress(
SDValue N, X86ISelAddressMode &AM) {
1891 if (matchAddressRecursively(
N, AM, 0))
1898 if (Subtarget->isTarget64BitILP32() &&
1899 AM.BaseType == X86ISelAddressMode::RegBase &&
1900 AM.Base_Reg.getNode() !=
nullptr && AM.IndexReg.getNode() ==
nullptr) {
1901 SDValue Save_Base_Reg = AM.Base_Reg;
1902 if (
auto *LoadN = dyn_cast<LoadSDNode>(Save_Base_Reg)) {
1904 if (matchLoadInAddress(LoadN, AM,
true))
1905 AM.Base_Reg = Save_Base_Reg;
1911 if (AM.Scale == 2 &&
1912 AM.BaseType == X86ISelAddressMode::RegBase &&
1913 AM.Base_Reg.getNode() ==
nullptr) {
1914 AM.Base_Reg = AM.IndexReg;
1921 (!AM.GV || !
TM.isLargeGlobalValue(AM.GV)) && Subtarget->is64Bit() &&
1922 AM.Scale == 1 && AM.BaseType == X86ISelAddressMode::RegBase &&
1923 AM.Base_Reg.getNode() ==
nullptr && AM.IndexReg.getNode() ==
nullptr &&
1925 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64);
1931bool X86DAGToDAGISel::matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
1937 X86ISelAddressMode Backup = AM;
1938 if (!matchAddressRecursively(
N.getOperand(0), AM,
Depth+1) &&
1939 !matchAddressRecursively(Handle.getValue().getOperand(1), AM,
Depth+1))
1944 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM,
1946 !matchAddressRecursively(Handle.getValue().getOperand(0), AM,
Depth + 1))
1953 if (AM.BaseType == X86ISelAddressMode::RegBase &&
1954 !AM.Base_Reg.getNode() &&
1955 !AM.IndexReg.getNode()) {
1956 N = Handle.getValue();
1957 AM.Base_Reg =
N.getOperand(0);
1958 AM.IndexReg =
N.getOperand(1);
1962 N = Handle.getValue();
1972 if (
N->getNodeId() == -1 ||
1992 X86ISelAddressMode &AM) {
1999 if (ScaleLog <= 0 || ScaleLog >= 4 ||
2000 Mask != (0xffu << ScaleLog))
2003 MVT XVT =
X.getSimpleValueType();
2004 MVT VT =
N.getSimpleValueType();
2029 AM.Scale = (1 << ScaleLog);
2037 X86ISelAddressMode &AM) {
2043 int64_t Mask = cast<ConstantSDNode>(
N->getOperand(1))->getSExtValue();
2048 bool FoundAnyExtend =
false;
2052 FoundAnyExtend =
true;
2070 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3)
2073 MVT VT =
N.getSimpleValueType();
2075 if (FoundAnyExtend) {
2096 AM.Scale = 1 << ShiftAmt;
2097 AM.IndexReg = NewAnd;
2131 X86ISelAddressMode &AM) {
2137 unsigned MaskIdx, MaskLen;
2140 unsigned MaskLZ = 64 - (MaskIdx + MaskLen);
2146 unsigned AMShiftAmt = MaskIdx;
2150 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2154 unsigned ScaleDown = (64 -
X.getSimpleValueType().getSizeInBits()) + ShiftAmt;
2155 if (MaskLZ < ScaleDown)
2157 MaskLZ -= ScaleDown;
2165 bool ReplacingAnyExtend =
false;
2167 unsigned ExtendBits =
X.getSimpleValueType().getSizeInBits() -
2168 X.getOperand(0).getSimpleValueType().getSizeInBits();
2171 X =
X.getOperand(0);
2172 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits;
2173 ReplacingAnyExtend =
true;
2175 APInt MaskedHighBits =
2182 MVT VT =
N.getSimpleValueType();
2183 if (ReplacingAnyExtend) {
2184 assert(
X.getValueType() != VT);
2191 MVT XVT =
X.getSimpleValueType();
2212 AM.Scale = 1 << AMShiftAmt;
2213 AM.IndexReg = NewExt;
2223 X86ISelAddressMode &AM,
2231 if (!Subtarget.hasTBM() &&
2232 !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR()))
2236 unsigned MaskIdx, MaskLen;
2244 unsigned AMShiftAmt = MaskIdx;
2248 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2250 MVT XVT =
X.getSimpleValueType();
2251 MVT VT =
N.getSimpleValueType();
2276 AM.Scale = 1 << AMShiftAmt;
2277 AM.IndexReg = NewExt;
2284 X86ISelAddressMode &AM,
2286 assert(AM.IndexReg.getNode() ==
nullptr &&
"IndexReg already matched");
2287 assert((AM.Scale == 1 || AM.Scale == 2 || AM.Scale == 4 || AM.Scale == 8) &&
2288 "Illegal index scale");
2294 EVT VT =
N.getValueType();
2295 unsigned Opc =
N.getOpcode();
2298 if (CurDAG->isBaseWithConstantOffset(
N)) {
2299 auto *AddVal = cast<ConstantSDNode>(
N.getOperand(1));
2301 if (!foldOffsetIntoAddress(
Offset, AM))
2302 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2306 if (Opc ==
ISD::ADD &&
N.getOperand(0) ==
N.getOperand(1)) {
2307 if (AM.Scale <= 4) {
2309 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2315 uint64_t ShiftAmt =
N.getConstantOperandVal(1);
2316 uint64_t ScaleAmt = 1ULL << ShiftAmt;
2317 if ((AM.Scale * ScaleAmt) <= 8) {
2318 AM.Scale *= ScaleAmt;
2319 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2327 if (Src.getOpcode() ==
ISD::ADD && Src->getFlags().hasNoSignedWrap() &&
2329 if (CurDAG->isBaseWithConstantOffset(Src)) {
2330 SDValue AddSrc = Src.getOperand(0);
2331 auto *AddVal = cast<ConstantSDNode>(Src.getOperand(1));
2333 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2335 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2341 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2342 CurDAG->RemoveDeadNode(
N.getNode());
2354 unsigned SrcOpc = Src.getOpcode();
2355 if (((SrcOpc ==
ISD::ADD && Src->getFlags().hasNoUnsignedWrap()) ||
2356 CurDAG->isADDLike(Src,
true)) &&
2358 if (CurDAG->isBaseWithConstantOffset(Src)) {
2359 SDValue AddSrc = Src.getOperand(0);
2361 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2372 if ((AM.Scale * ScaleAmt) <= 8 &&
2374 CurDAG->MaskedValueIsZero(ShVal, HiBits))) {
2375 AM.Scale *= ScaleAmt;
2376 SDValue ExtShVal = CurDAG->getNode(Opc,
DL, VT, ShVal);
2385 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2387 SDValue ExtAdd = CurDAG->getNode(SrcOpc,
DL, VT, ExtSrc, ExtVal);
2391 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2392 CurDAG->RemoveDeadNode(
N.getNode());
2393 return Res ? Res : ExtSrc;
2403bool X86DAGToDAGISel::matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
2407 dbgs() <<
"MatchAddress: ";
2412 return matchAddressBase(
N, AM);
2417 if (AM.isRIPRelative()) {
2421 if (!(AM.ES || AM.MCSym) && AM.JT != -1)
2424 if (
auto *Cst = dyn_cast<ConstantSDNode>(
N))
2425 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM))
2430 switch (
N.getOpcode()) {
2433 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
2434 if (
const auto *ESNode = dyn_cast<MCSymbolSDNode>(
N.getOperand(0))) {
2436 AM.MCSym = ESNode->getMCSymbol();
2442 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2443 if (!foldOffsetIntoAddress(Val, AM))
2450 if (!matchWrapper(
N, AM))
2455 if (!matchLoadInAddress(cast<LoadSDNode>(
N), AM))
2460 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2461 AM.Base_Reg.getNode() ==
nullptr &&
2463 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
2464 AM.Base_FrameIndex = cast<FrameIndexSDNode>(
N)->getIndex();
2470 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2473 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
2474 unsigned Val = CN->getZExtValue();
2479 if (Val == 1 || Val == 2 || Val == 3) {
2481 AM.Scale = 1 << Val;
2482 AM.IndexReg = matchIndexRecursively(ShVal, AM,
Depth + 1);
2490 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2494 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2495 "Unexpected value size!");
2504 if (!isa<ConstantSDNode>(
N.getOperand(1)) ||
2505 !isa<ConstantSDNode>(
And.getOperand(1)))
2507 uint64_t Mask =
And.getConstantOperandVal(1) >>
N.getConstantOperandVal(1);
2519 if (
N.getResNo() != 0)
break;
2524 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2525 AM.Base_Reg.getNode() ==
nullptr &&
2526 AM.IndexReg.getNode() ==
nullptr) {
2527 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
2528 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
2529 CN->getZExtValue() == 9) {
2530 AM.Scale =
unsigned(CN->getZExtValue())-1;
2541 auto *AddVal = cast<ConstantSDNode>(MulVal.
getOperand(1));
2542 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue();
2543 if (foldOffsetIntoAddress(Disp, AM))
2544 Reg =
N.getOperand(0);
2546 Reg =
N.getOperand(0);
2549 AM.IndexReg = AM.Base_Reg =
Reg;
2568 X86ISelAddressMode Backup = AM;
2569 if (matchAddressRecursively(
N.getOperand(0), AM,
Depth+1)) {
2570 N = Handle.getValue();
2574 N = Handle.getValue();
2576 if (AM.IndexReg.getNode() || AM.isRIPRelative()) {
2591 RHS.getOperand(0).getValueType() == MVT::i32))
2595 if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() &&
2596 !AM.Base_Reg.getNode()->hasOneUse()) ||
2597 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
2601 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
2602 ((AM.Disp != 0) && (Backup.Disp == 0)) +
2603 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
2615 AM.NegateIndex =
true;
2623 if (!CurDAG->isADDLike(
N))
2627 if (!matchAdd(
N, AM,
Depth))
2636 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2640 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2641 "Unexpected value size!");
2643 if (!isa<ConstantSDNode>(
N.getOperand(1)))
2646 if (
N.getOperand(0).getOpcode() ==
ISD::SRL) {
2675 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2685 AM.IndexReg =
Index;
2691 if (Src.getOpcode() ==
ISD::AND && Src.hasOneUse())
2692 if (
auto *MaskC = dyn_cast<ConstantSDNode>(Src.getOperand(1))) {
2693 Mask = MaskC->getAPIntValue();
2694 Src = Src.getOperand(0);
2697 if (Src.getOpcode() ==
ISD::SHL && Src.hasOneUse()) {
2699 SDValue ShlSrc = Src.getOperand(0);
2700 SDValue ShlAmt = Src.getOperand(1);
2701 auto *ShAmtC = dyn_cast<ConstantSDNode>(ShlAmt);
2704 unsigned ShAmtV = ShAmtC->getZExtValue();
2712 if (!Src->getFlags().hasNoUnsignedWrap() &&
2713 !CurDAG->MaskedValueIsZero(ShlSrc, HighZeros & Mask))
2721 MVT VT =
N.getSimpleValueType();
2725 if (!
Mask.isAllOnes()) {
2726 Res = CurDAG->getConstant(
Mask.lshr(ShAmtV),
DL, SrcVT);
2728 Res = CurDAG->getNode(
ISD::AND,
DL, SrcVT, ShlSrc, Res);
2735 CurDAG->ReplaceAllUsesWith(
N, NewShl);
2736 CurDAG->RemoveDeadNode(
N.getNode());
2739 AM.Scale = 1 << ShAmtV;
2743 AM.IndexReg = matchIndexRecursively(Zext, AM,
Depth + 1);
2747 if (Src.getOpcode() ==
ISD::SRL && !
Mask.isAllOnes()) {
2750 Src.getOperand(0), AM))
2755 Src.getOperand(0), AM))
2760 Src.getOperand(0), AM, *Subtarget))
2768 return matchAddressBase(
N, AM);
2773bool X86DAGToDAGISel::matchAddressBase(
SDValue N, X86ISelAddressMode &AM) {
2775 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
2777 if (!AM.IndexReg.getNode()) {
2788 AM.BaseType = X86ISelAddressMode::RegBase;
2793bool X86DAGToDAGISel::matchVectorAddressRecursively(
SDValue N,
2794 X86ISelAddressMode &AM,
2798 dbgs() <<
"MatchVectorAddress: ";
2803 return matchAddressBase(
N, AM);
2806 switch (
N.getOpcode()) {
2808 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2809 if (!foldOffsetIntoAddress(Val, AM))
2814 if (!matchWrapper(
N, AM))
2822 X86ISelAddressMode Backup = AM;
2823 if (!matchVectorAddressRecursively(
N.getOperand(0), AM,
Depth + 1) &&
2824 !matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2830 if (!matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2832 !matchVectorAddressRecursively(Handle.getValue().getOperand(0), AM,
2837 N = Handle.getValue();
2842 return matchAddressBase(
N, AM);
2848bool X86DAGToDAGISel::matchVectorAddress(
SDValue N, X86ISelAddressMode &AM) {
2849 return matchVectorAddressRecursively(
N, AM, 0);
2857 X86ISelAddressMode AM;
2863 AM.IndexReg = matchIndexRecursively(IndexOp, AM, 0);
2865 AM.IndexReg = IndexOp;
2869 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2871 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2873 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2879 if (matchVectorAddress(BasePtr, AM))
2882 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2896 X86ISelAddressMode AM;
2908 unsigned AddrSpace =
2909 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
2911 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2913 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2915 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2920 MVT VT =
N.getSimpleValueType();
2922 if (matchAddress(
N, AM))
2925 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2941 N =
N.getOperand(0);
2956 const GlobalValue *GV = cast<GlobalAddressSDNode>(
N)->getGlobal();
2958 return CR->getUnsignedMax().ult(1ull << 32);
2960 return !
TM.isLargeGlobalValue(GV);
2969 if (!selectLEAAddr(
N,
Base, Scale,
Index, Disp, Segment))
2972 auto *
RN = dyn_cast<RegisterSDNode>(
Base);
2973 if (RN &&
RN->getReg() == 0)
2974 Base = CurDAG->getRegister(0, MVT::i64);
2975 else if (
Base.getValueType() == MVT::i32 && !isa<FrameIndexSDNode>(
Base)) {
2979 Base = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
2983 RN = dyn_cast<RegisterSDNode>(
Index);
2984 if (RN &&
RN->getReg() == 0)
2985 Index = CurDAG->getRegister(0, MVT::i64);
2988 "Expect to be extending 32-bit registers for use in LEA");
2991 Index = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
3000bool X86DAGToDAGISel::selectLEAAddr(
SDValue N,
3004 X86ISelAddressMode AM;
3008 MVT VT =
N.getSimpleValueType();
3013 SDValue T = CurDAG->getRegister(0, MVT::i32);
3015 if (matchAddress(
N, AM))
3020 unsigned Complexity = 0;
3021 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode())
3023 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
3026 if (AM.IndexReg.getNode())
3039 if (AM.hasSymbolicDisplacement()) {
3041 if (Subtarget->is64Bit())
3051 auto isMathWithFlags = [](
SDValue V) {
3052 switch (
V.getOpcode()) {
3073 if (isMathWithFlags(
N.getOperand(0)) || isMathWithFlags(
N.getOperand(1)))
3081 if (Complexity <= 2)
3084 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
3095 X86ISelAddressMode AM;
3096 if (
auto *GA = dyn_cast<GlobalAddressSDNode>(
N)) {
3097 AM.GV = GA->getGlobal();
3098 AM.Disp += GA->getOffset();
3099 AM.SymbolFlags = GA->getTargetFlags();
3101 auto *SA = cast<ExternalSymbolSDNode>(
N);
3102 AM.ES = SA->getSymbol();
3103 AM.SymbolFlags = SA->getTargetFlags();
3106 if (Subtarget->is32Bit()) {
3108 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
3111 MVT VT =
N.getSimpleValueType();
3112 getAddressOperands(AM,
SDLoc(
N), VT,
Base, Scale,
Index, Disp, Segment);
3120 EVT VT =
N.getValueType();
3121 bool WasTruncated =
false;
3123 WasTruncated =
true;
3124 N =
N.getOperand(0);
3133 unsigned Opc =
N.getOperand(0)->getOpcode();
3135 Op =
N.getOperand(0);
3138 return !WasTruncated;
3142 auto *GA = cast<GlobalAddressSDNode>(
N.getOperand(0));
3143 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange();
3144 if (!CR || CR->getUnsignedMax().uge(1ull << VT.
getSizeInBits()))
3148 Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(),
SDLoc(
N), VT,
3149 GA->getOffset(), GA->getTargetFlags());
3157 assert(Root &&
P &&
"Unknown root/parent nodes");
3159 !IsProfitableToFold(
N,
P, Root) ||
3160 !IsLegalToFold(
N,
P, Root, OptLevel))
3163 return selectAddr(
N.getNode(),
3164 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3171 assert(Root &&
P &&
"Unknown root/parent nodes");
3173 !IsProfitableToFold(
N,
P, Root) ||
3174 !IsLegalToFold(
N,
P, Root, OptLevel))
3177 return selectAddr(
N.getNode(),
3178 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3184SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
3185 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
3187 return CurDAG->getRegister(GlobalBaseReg, TLI->
getPointerTy(
DL)).getNode();
3190bool X86DAGToDAGISel::isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const {
3192 N =
N->getOperand(0).getNode();
3196 auto *GA = dyn_cast<GlobalAddressSDNode>(
N->getOperand(0));
3200 auto *GV = GA->getGlobal();
3203 return CR->getSignedMin().sge(-1ull << Width) &&
3204 CR->getSignedMax().slt(1ull << Width);
3209 return Width == 32 && !
TM.isLargeGlobalValue(GV);
3213 assert(
N->isMachineOpcode() &&
"Unexpected node");
3214 unsigned Opc =
N->getMachineOpcode();
3215 const MCInstrDesc &MCID = getInstrInfo()->get(Opc);
3220 return static_cast<X86::CondCode>(
N->getConstantOperandVal(CondNo));
3225bool X86DAGToDAGISel::onlyUsesZeroFlag(
SDValue Flags)
const {
3230 if (UI.getUse().getResNo() !=
Flags.getResNo())
3234 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3238 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3240 if (FlagUI.getUse().getResNo() != 1)
continue;
3242 if (!FlagUI->isMachineOpcode())
return false;
3261bool X86DAGToDAGISel::hasNoSignFlagUses(
SDValue Flags)
const {
3266 if (UI.getUse().getResNo() !=
Flags.getResNo())
3270 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3274 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3276 if (FlagUI.getUse().getResNo() != 1)
continue;
3278 if (!FlagUI->isMachineOpcode())
return false;
3317 bool X86DAGToDAGISel::hasNoCarryFlagUses(
SDValue Flags)
const {
3322 if (UI.getUse().getResNo() !=
Flags.getResNo())
3325 unsigned UIOpc = UI->getOpcode();
3329 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3333 FlagUI != FlagUE; ++FlagUI) {
3335 if (FlagUI.getUse().getResNo() != 1)
3338 if (!FlagUI->isMachineOpcode())
3379 if (StoredVal.
getResNo() != 0)
return false;
3393 LoadNode = cast<LoadSDNode>(Load);
3396 if (!Load.hasOneUse())
3404 bool FoundLoad =
false;
3408 const unsigned int Max = 1024;
3450 if (Chain == Load.getValue(1)) {
3456 if (
Op == Load.getValue(1)) {
3472 if (
Op.getNode() != LoadNode)
3505bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(
SDNode *
Node) {
3506 auto *StoreNode = cast<StoreSDNode>(
Node);
3513 EVT MemVT = StoreNode->getMemoryVT();
3514 if (MemVT != MVT::i64 && MemVT != MVT::i32 && MemVT != MVT::i16 &&
3518 bool IsCommutable =
false;
3519 bool IsNegate =
false;
3533 IsCommutable =
true;
3537 unsigned LoadOpNo = IsNegate ? 1 : 0;
3541 LoadNode, InputChain)) {
3548 LoadNode, InputChain))
3557 auto SelectOpcode = [&](
unsigned Opc64,
unsigned Opc32,
unsigned Opc16,
3578 unsigned NewOpc = SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m,
3588 if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) {
3592 if ((IsOne || IsNegOne) && hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3595 ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
3596 : SelectOpcode(
X86::DEC64m,
X86::DEC32m,
X86::DEC16m,
X86::DEC8m);
3609 auto SelectRegOpcode = [SelectOpcode](
unsigned Opc) {
3612 return SelectOpcode(X86::ADD64mr, X86::ADD32mr, X86::ADD16mr,
3615 return SelectOpcode(X86::ADC64mr, X86::ADC32mr, X86::ADC16mr,
3618 return SelectOpcode(X86::SUB64mr, X86::SUB32mr, X86::SUB16mr,
3621 return SelectOpcode(X86::SBB64mr, X86::SBB32mr, X86::SBB16mr,
3624 return SelectOpcode(X86::AND64mr, X86::AND32mr, X86::AND16mr,
3627 return SelectOpcode(X86::OR64mr, X86::OR32mr, X86::OR16mr, X86::OR8mr);
3629 return SelectOpcode(X86::XOR64mr, X86::XOR32mr, X86::XOR16mr,
3635 auto SelectImmOpcode = [SelectOpcode](
unsigned Opc) {
3638 return SelectOpcode(X86::ADD64mi32, X86::ADD32mi, X86::ADD16mi,
3641 return SelectOpcode(X86::ADC64mi32, X86::ADC32mi, X86::ADC16mi,
3644 return SelectOpcode(X86::SUB64mi32, X86::SUB32mi, X86::SUB16mi,
3647 return SelectOpcode(X86::SBB64mi32, X86::SBB32mi, X86::SBB16mi,
3650 return SelectOpcode(X86::AND64mi32, X86::AND32mi, X86::AND16mi,
3653 return SelectOpcode(X86::OR64mi32, X86::OR32mi, X86::OR16mi,
3656 return SelectOpcode(X86::XOR64mi32, X86::XOR32mi, X86::XOR16mi,
3663 unsigned NewOpc = SelectRegOpcode(Opc);
3668 if (
auto *OperandC = dyn_cast<ConstantSDNode>(Operand)) {
3669 int64_t OperandV = OperandC->getSExtValue();
3675 ((MemVT != MVT::i8 && !isInt<8>(OperandV) && isInt<8>(-OperandV)) ||
3676 (MemVT == MVT::i64 && !isInt<32>(OperandV) &&
3677 isInt<32>(-OperandV))) &&
3678 hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3679 OperandV = -OperandV;
3683 if (MemVT != MVT::i64 || isInt<32>(OperandV)) {
3684 Operand = CurDAG->getTargetConstant(OperandV,
SDLoc(
Node), MemVT);
3685 NewOpc = SelectImmOpcode(Opc);
3691 CurDAG->getCopyToReg(InputChain,
SDLoc(
Node), X86::EFLAGS,
3695 Segment, Operand, CopyTo, CopyTo.
getValue(1)};
3696 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3700 Segment, Operand, InputChain};
3701 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3712 CurDAG->setNodeMemRefs(Result, MemOps);
3718 CurDAG->RemoveDeadNode(
Node);
3729bool X86DAGToDAGISel::matchBitExtract(
SDNode *
Node) {
3733 "Should be either an and-mask, or right-shift after clearing high bits.");
3736 if (!Subtarget->hasBMI() && !Subtarget->hasBMI2())
3739 MVT NVT =
Node->getSimpleValueType(0);
3742 if (NVT != MVT::i32 && NVT != MVT::i64)
3750 const bool AllowExtraUsesByDefault = Subtarget->hasBMI2();
3751 auto checkUses = [AllowExtraUsesByDefault](
3753 std::optional<bool> AllowExtraUses) {
3754 return AllowExtraUses.value_or(AllowExtraUsesByDefault) ||
3755 Op.getNode()->hasNUsesOfValue(NUses,
Op.getResNo());
3757 auto checkOneUse = [checkUses](
SDValue Op,
3758 std::optional<bool> AllowExtraUses =
3760 return checkUses(
Op, 1, AllowExtraUses);
3762 auto checkTwoUse = [checkUses](
SDValue Op,
3763 std::optional<bool> AllowExtraUses =
3765 return checkUses(
Op, 2, AllowExtraUses);
3768 auto peekThroughOneUseTruncation = [checkOneUse](
SDValue V) {
3770 assert(
V.getSimpleValueType() == MVT::i32 &&
3771 V.getOperand(0).getSimpleValueType() == MVT::i64 &&
3772 "Expected i64 -> i32 truncation");
3773 V =
V.getOperand(0);
3779 auto matchPatternA = [checkOneUse, peekThroughOneUseTruncation, &NBits,
3782 if (
Mask->getOpcode() !=
ISD::ADD || !checkOneUse(Mask))
3788 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3793 NBits =
M0->getOperand(1);
3794 NegateNBits =
false;
3798 auto isAllOnes = [
this, peekThroughOneUseTruncation, NVT](
SDValue V) {
3799 V = peekThroughOneUseTruncation(V);
3800 return CurDAG->MaskedValueIsAllOnes(
3806 auto matchPatternB = [checkOneUse, isAllOnes, peekThroughOneUseTruncation,
3809 if (
Mask.getOpcode() !=
ISD::XOR || !checkOneUse(Mask))
3812 if (!isAllOnes(
Mask->getOperand(1)))
3815 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3819 if (!isAllOnes(
M0->getOperand(0)))
3821 NBits =
M0->getOperand(1);
3822 NegateNBits =
false;
3828 auto canonicalizeShiftAmt = [&NBits, &NegateNBits](
SDValue ShiftAmt,
3829 unsigned Bitwidth) {
3834 NBits = NBits.getOperand(0);
3839 auto *V0 = dyn_cast<ConstantSDNode>(NBits.getOperand(0));
3840 if (!V0 || V0->getZExtValue() != Bitwidth)
3842 NBits = NBits.getOperand(1);
3843 NegateNBits =
false;
3849 auto matchPatternC = [checkOneUse, peekThroughOneUseTruncation, &NegateNBits,
3852 Mask = peekThroughOneUseTruncation(Mask);
3853 unsigned Bitwidth =
Mask.getSimpleValueType().getSizeInBits();
3855 if (
Mask.getOpcode() !=
ISD::SRL || !checkOneUse(Mask))
3862 if (!checkOneUse(
M1))
3864 canonicalizeShiftAmt(
M1, Bitwidth);
3869 return !NegateNBits;
3877 auto matchPatternD = [checkOneUse, checkTwoUse, canonicalizeShiftAmt,
3878 AllowExtraUsesByDefault, &NegateNBits,
3891 canonicalizeShiftAmt(N1, Bitwidth);
3895 const bool AllowExtraUses = AllowExtraUsesByDefault && !NegateNBits;
3896 if (!checkOneUse(N0, AllowExtraUses) || !checkTwoUse(N1, AllowExtraUses))
3902 auto matchLowBitMask = [matchPatternA, matchPatternB,
3904 return matchPatternA(Mask) || matchPatternB(Mask) || matchPatternC(Mask);
3908 X =
Node->getOperand(0);
3911 if (matchLowBitMask(Mask)) {
3915 if (!matchLowBitMask(Mask))
3919 X = CurDAG->getAllOnesConstant(
SDLoc(
Node), NVT);
3920 }
else if (!matchPatternD(
Node))
3925 if (NegateNBits && !Subtarget->hasBMI2())
3937 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, MVT::i32), 0);
3940 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit,
DL, MVT::i32);
3942 NBits =
SDValue(CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG,
DL,
3943 MVT::i32, ImplDef, NBits, SRIdxVal),
3953 NBits = CurDAG->getNode(
ISD::SUB,
DL, MVT::i32, BitWidthC, NBits);
3957 if (Subtarget->hasBMI2()) {
3959 if (NVT != MVT::i32) {
3967 SelectCode(Extract.
getNode());
3976 SDValue RealX = peekThroughOneUseTruncation(
X);
3982 MVT XVT =
X.getSimpleValueType();
3992 SDValue C8 = CurDAG->getConstant(8,
DL, MVT::i8);
4000 SDValue ShiftAmt =
X.getOperand(1);
4001 X =
X.getOperand(0);
4004 "Expected shift amount to be i8");
4008 SDValue OrigShiftAmt = ShiftAmt;
4013 Control = CurDAG->getNode(
ISD::OR,
DL, MVT::i32, Control, ShiftAmt);
4018 if (XVT != MVT::i32) {
4033 SelectCode(Extract.
getNode());
4040 MVT NVT =
Node->getSimpleValueType(0);
4053 Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR());
4054 if (!PreferBEXTR && !Subtarget->hasBMI2())
4066 if (NVT != MVT::i32 && NVT != MVT::i64)
4070 auto *MaskCst = dyn_cast<ConstantSDNode>(N1);
4071 auto *ShiftCst = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
4072 if (!MaskCst || !ShiftCst)
4080 uint64_t Shift = ShiftCst->getZExtValue();
4085 if (Shift == 8 && MaskSize == 8)
4096 if (!PreferBEXTR && MaskSize <= 32)
4100 unsigned ROpc, MOpc;
4102#define GET_EGPR_IF_ENABLED(OPC) (Subtarget->hasEGPR() ? OPC##_EVEX : OPC)
4104 assert(Subtarget->hasBMI2() &&
"We must have BMI2's BZHI then.");
4108 Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT);
4113 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4114 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4120 Control = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT);
4121 if (Subtarget->hasTBM()) {
4122 ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri;
4123 MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi;
4125 assert(Subtarget->hasBMI() &&
"We must have BMI1's BEXTR then.");
4131 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4132 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4138 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4139 if (tryFoldLoad(
Node, N0.
getNode(), Input, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4141 Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Control, Input.
getOperand(0)};
4142 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
4143 NewNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4147 CurDAG->setNodeMemRefs(NewNode, {cast<LoadSDNode>(Input)->getMemOperand()});
4149 NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, Control);
4154 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT);
4158 CurDAG->getMachineNode(NewOpc, dl, NVT,
SDValue(NewNode, 0), ShAmt);
4165MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
4166 bool MayFoldLoad,
const SDLoc &dl,
4171 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4172 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4175 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4176 if (MayFoldLoad && tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4177 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4179 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other);
4180 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4184 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
4189 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32);
4190 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4197MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
4198 bool MayFoldLoad,
const SDLoc &dl,
4204 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4205 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4208 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4209 if (MayFoldLoad && tryFoldLoad(
Node, N2, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4210 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4212 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other, MVT::Glue);
4213 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4218 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N2)->getMemOperand()});
4223 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Glue);
4224 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4229bool X86DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
4230 EVT VT =
N->getValueType(0);
4237 unsigned Size = VT == MVT::i64 ? 64 : 32;
4239 SDValue OrigShiftAmt =
N->getOperand(1);
4240 SDValue ShiftAmt = OrigShiftAmt;
4255 auto *Add0C = dyn_cast<ConstantSDNode>(Add0);
4256 auto *Add1C = dyn_cast<ConstantSDNode>(Add1);
4259 if (Add1C && Add1C->getAPIntValue().urem(
Size) == 0) {
4263 ((Add0C && Add0C->getAPIntValue().urem(
Size) ==
Size - 1) ||
4264 (Add1C && Add1C->getAPIntValue().urem(
Size) ==
Size - 1))) {
4268 assert(Add0C ==
nullptr || Add1C ==
nullptr);
4277 NewShiftAmt = CurDAG->getNode(
ISD::XOR,
DL, OpVT,
4278 Add0C ==
nullptr ? Add0 : Add1,
AllOnes);
4284 Add0C->getZExtValue() != 0) {
4287 if (Add0C->getZExtValue() %
Size == 0)
4290 Add0C->getZExtValue() % 32 == 0) {
4298 Add0 = CurDAG->getZExtOrTrunc(Add0,
DL, SubVT);
4302 X = CurDAG->getNode(
ISD::ADD,
DL, SubVT, Add1, Add0);
4324 NewShiftAmt = CurDAG->getNode(
ISD::TRUNCATE,
DL, MVT::i8, NewShiftAmt);
4331 NewShiftAmt = CurDAG->getNode(
ISD::AND,
DL, MVT::i8, NewShiftAmt,
4332 CurDAG->getConstant(
Size - 1,
DL, MVT::i8));
4336 SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(
N,
N->getOperand(0),
4338 if (UpdatedNode !=
N) {
4341 ReplaceNode(
N, UpdatedNode);
4348 CurDAG->RemoveDeadNode(OrigShiftAmt.
getNode());
4356bool X86DAGToDAGISel::tryShrinkShlLogicImm(
SDNode *
N) {
4357 MVT NVT =
N->getSimpleValueType(0);
4358 unsigned Opcode =
N->getOpcode();
4366 auto *Cst = dyn_cast<ConstantSDNode>(N1);
4370 int64_t Val = Cst->getSExtValue();
4375 bool FoundAnyExtend =
false;
4379 FoundAnyExtend =
true;
4387 if (NVT != MVT::i32 && NVT != MVT::i64)
4390 auto *ShlCst = dyn_cast<ConstantSDNode>(Shift.
getOperand(1));
4394 uint64_t ShAmt = ShlCst->getZExtValue();
4398 uint64_t RemovedBitsMask = (1ULL << ShAmt) - 1;
4399 if (Opcode !=
ISD::AND && (Val & RemovedBitsMask) != 0)
4404 auto CanShrinkImmediate = [&](int64_t &ShiftedVal) {
4408 ShiftedVal = (
uint64_t)Val >> ShAmt;
4409 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4412 if (ShiftedVal == UINT8_MAX || ShiftedVal == UINT16_MAX)
4415 ShiftedVal = Val >> ShAmt;
4416 if ((!isInt<8>(Val) && isInt<8>(ShiftedVal)) ||
4417 (!isInt<32>(Val) && isInt<32>(ShiftedVal)))
4421 ShiftedVal = (
uint64_t)Val >> ShAmt;
4422 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4429 if (!CanShrinkImmediate(ShiftedVal))
4439 unsigned ZExtWidth = Cst->getAPIntValue().getActiveBits();
4445 NeededMask &= ~Cst->getAPIntValue();
4447 if (CurDAG->MaskedValueIsZero(
N->getOperand(0), NeededMask))
4452 if (FoundAnyExtend) {
4458 SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT);
4460 SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT,
X, NewCst);
4469bool X86DAGToDAGISel::matchVPTERNLOG(
SDNode *Root,
SDNode *ParentA,
4473 assert(
A.isOperandOf(ParentA) &&
B.isOperandOf(ParentB) &&
4474 C.isOperandOf(ParentC) &&
"Incorrect parent node");
4476 auto tryFoldLoadOrBCast =
4479 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4485 L =
L.getOperand(0);
4492 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4493 unsigned Size = MemIntr->getMemoryVT().getSizeInBits();
4497 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4500 bool FoldedLoad =
false;
4501 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4502 if (tryFoldLoadOrBCast(Root, ParentC,
C, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4504 }
else if (tryFoldLoadOrBCast(Root, ParentA,
A, Tmp0, Tmp1, Tmp2, Tmp3,
4509 uint8_t OldImm =
Imm;
4510 Imm = OldImm & 0xa5;
4511 if (OldImm & 0x02)
Imm |= 0x10;
4512 if (OldImm & 0x10)
Imm |= 0x02;
4513 if (OldImm & 0x08)
Imm |= 0x40;
4514 if (OldImm & 0x40)
Imm |= 0x08;
4515 }
else if (tryFoldLoadOrBCast(Root, ParentB,
B, Tmp0, Tmp1, Tmp2, Tmp3,
4520 uint8_t OldImm =
Imm;
4521 Imm = OldImm & 0x99;
4522 if (OldImm & 0x02)
Imm |= 0x04;
4523 if (OldImm & 0x04)
Imm |= 0x02;
4524 if (OldImm & 0x20)
Imm |= 0x40;
4525 if (OldImm & 0x40)
Imm |= 0x20;
4530 SDValue TImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
4536 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
4540 auto *MemIntr = cast<MemIntrinsicSDNode>(
C);
4541 unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits();
4542 assert((EltSize == 32 || EltSize == 64) &&
"Unexpected broadcast size!");
4544 bool UseD = EltSize == 32;
4546 Opc = UseD ? X86::VPTERNLOGDZ128rmbi : X86::VPTERNLOGQZ128rmbi;
4548 Opc = UseD ? X86::VPTERNLOGDZ256rmbi : X86::VPTERNLOGQZ256rmbi;
4550 Opc = UseD ? X86::VPTERNLOGDZrmbi : X86::VPTERNLOGQZrmbi;
4556 Opc = UseD ? X86::VPTERNLOGDZ128rmi : X86::VPTERNLOGQZ128rmi;
4558 Opc = UseD ? X86::VPTERNLOGDZ256rmi : X86::VPTERNLOGQZ256rmi;
4560 Opc = UseD ? X86::VPTERNLOGDZrmi : X86::VPTERNLOGQZrmi;
4565 SDValue Ops[] = {
A,
B, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, TImm,
C.getOperand(0)};
4566 MNode = CurDAG->getMachineNode(Opc,
DL, VTs, Ops);
4569 ReplaceUses(
C.getValue(1),
SDValue(MNode, 1));
4571 CurDAG->setNodeMemRefs(MNode, {cast<MemSDNode>(
C)->getMemOperand()});
4576 Opc = UseD ? X86::VPTERNLOGDZ128rri : X86::VPTERNLOGQZ128rri;
4578 Opc = UseD ? X86::VPTERNLOGDZ256rri : X86::VPTERNLOGQZ256rri;
4580 Opc = UseD ? X86::VPTERNLOGDZrri : X86::VPTERNLOGQZrri;
4584 MNode = CurDAG->getMachineNode(Opc,
DL, NVT, {
A,
B,
C, TImm});
4588 CurDAG->RemoveDeadNode(Root);
4594bool X86DAGToDAGISel::tryVPTERNLOG(
SDNode *
N) {
4595 MVT NVT =
N->getSimpleValueType(0);
4598 if (!NVT.
isVector() || !Subtarget->hasAVX512() ||
4609 auto getFoldableLogicOp = [](
SDValue Op) {
4612 Op =
Op.getOperand(0);
4614 if (!
Op.hasOneUse())
4617 unsigned Opc =
Op.getOpcode();
4626 if ((FoldableOp = getFoldableLogicOp(N1))) {
4628 }
else if ((FoldableOp = getFoldableLogicOp(N0))) {
4641 uint8_t TernlogMagicA = 0xf0;
4642 uint8_t TernlogMagicB = 0xcc;
4643 uint8_t TernlogMagicC = 0xaa;
4652 Parent =
Op.getNode();
4653 Op =
Op.getOperand(0);
4657 PeekThroughNot(
A, ParentA, TernlogMagicA);
4658 PeekThroughNot(
B, ParentB, TernlogMagicB);
4659 PeekThroughNot(
C, ParentC, TernlogMagicC);
4664 case ISD::AND:
Imm = TernlogMagicB & TernlogMagicC;
break;
4665 case ISD::OR:
Imm = TernlogMagicB | TernlogMagicC;
break;
4666 case ISD::XOR:
Imm = TernlogMagicB ^ TernlogMagicC;
break;
4670 switch (
N->getOpcode()) {
4674 Imm &= ~TernlogMagicA;
4676 Imm = ~(
Imm) & TernlogMagicA;
4683 return matchVPTERNLOG(
N, ParentA, ParentB, ParentC,
A,
B,
C, Imm);
4693bool X86DAGToDAGISel::shrinkAndImmediate(
SDNode *
And) {
4696 MVT VT =
And->getSimpleValueType(0);
4697 if (VT != MVT::i32 && VT != MVT::i64)
4700 auto *And1C = dyn_cast<ConstantSDNode>(
And->getOperand(1));
4709 APInt MaskVal = And1C->getAPIntValue();
4711 if (!MaskLZ || (VT == MVT::i64 && MaskLZ == 32))
4715 if (VT == MVT::i64 && MaskLZ >= 32) {
4717 MaskVal = MaskVal.
trunc(32);
4722 APInt NegMaskVal = MaskVal | HighZeros;
4731 if (VT == MVT::i64 && MaskVal.
getBitWidth() < 64) {
4732 NegMaskVal = NegMaskVal.
zext(64);
4733 HighZeros = HighZeros.
zext(64);
4738 if (!CurDAG->MaskedValueIsZero(And0, HighZeros))
4758 bool FoldedBCast,
bool Masked) {
4759#define VPTESTM_CASE(VT, SUFFIX) \
4762 return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
4763 return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
4766#define VPTESTM_BROADCAST_CASES(SUFFIX) \
4767default: llvm_unreachable("Unexpected VT!"); \
4768VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
4769VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
4770VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
4771VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
4772VPTESTM_CASE(v16i32, DZ##SUFFIX) \
4773VPTESTM_CASE(v8i64, QZ##SUFFIX)
4775#define VPTESTM_FULL_CASES(SUFFIX) \
4776VPTESTM_BROADCAST_CASES(SUFFIX) \
4777VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
4778VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
4779VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
4780VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
4781VPTESTM_CASE(v64i8, BZ##SUFFIX) \
4782VPTESTM_CASE(v32i16, WZ##SUFFIX)
4800#undef VPTESTM_FULL_CASES
4801#undef VPTESTM_BROADCAST_CASES
4807bool X86DAGToDAGISel::tryVPTESTM(
SDNode *Root,
SDValue Setcc,
4809 assert(Subtarget->hasAVX512() &&
"Expected AVX512!");
4859 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4864 if (CmpSVT != MVT::i32 && CmpSVT != MVT::i64)
4870 L =
L.getOperand(0);
4876 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4877 if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.
getSizeInBits())
4880 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4884 bool CanFoldLoads = Src0 != Src1;
4886 bool FoldedLoad =
false;
4887 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4889 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src1, Tmp0, Tmp1, Tmp2,
4893 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src0, Tmp0, Tmp1,
4902 bool IsMasked = InMask.
getNode() !=
nullptr;
4915 SDValue ImplDef =
SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl,
4917 Src0 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src0);
4920 Src1 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src1);
4925 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4926 InMask =
SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4927 dl, MaskVT, InMask, RC), 0);
4932 unsigned Opc =
getVPTESTMOpc(CmpVT, IsTestN, FoldedLoad, FoldedBCast,
4937 SDVTList VTs = CurDAG->getVTList(MaskVT, MVT::Other);
4940 SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4942 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
4944 SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4946 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
4952 CurDAG->setNodeMemRefs(CNode, {cast<MemSDNode>(Src1)->getMemOperand()});
4955 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, InMask, Src0, Src1);
4957 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, Src0, Src1);
4963 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4964 CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4965 dl, ResVT,
SDValue(CNode, 0), RC);
4969 CurDAG->RemoveDeadNode(Root);
4975bool X86DAGToDAGISel::tryMatchBitSelect(
SDNode *
N) {
4978 MVT NVT =
N->getSimpleValueType(0);
4981 if (!NVT.
isVector() || !Subtarget->hasAVX512())
5015 SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8);
5024 MVT NVT =
Node->getSimpleValueType(0);
5025 unsigned Opcode =
Node->getOpcode();
5028 if (
Node->isMachineOpcode()) {
5030 Node->setNodeId(-1);
5037 unsigned IntNo =
Node->getConstantOperandVal(1);
5040 case Intrinsic::x86_encodekey128:
5041 case Intrinsic::x86_encodekey256: {
5042 if (!Subtarget->hasKL())
5048 case Intrinsic::x86_encodekey128:
5049 Opcode = X86::ENCODEKEY128;
5051 case Intrinsic::x86_encodekey256:
5052 Opcode = X86::ENCODEKEY256;
5057 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(3),
5059 if (Opcode == X86::ENCODEKEY256)
5060 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(4),
5064 Opcode, dl,
Node->getVTList(),
5065 {Node->getOperand(2), Chain, Chain.getValue(1)});
5066 ReplaceNode(
Node, Res);
5069 case Intrinsic::x86_tileloadd64_internal:
5070 case Intrinsic::x86_tileloaddt164_internal: {
5071 if (!Subtarget->hasAMXTILE())
5073 unsigned Opc = IntNo == Intrinsic::x86_tileloadd64_internal
5075 : X86::PTILELOADDT1V;
5078 SDValue Scale = getI8Imm(1, dl);
5080 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5081 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5085 Node->getOperand(3),
5092 CNode = CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops);
5093 ReplaceNode(
Node, CNode);
5100 unsigned IntNo =
Node->getConstantOperandVal(1);
5103 case Intrinsic::x86_sse3_monitor:
5104 case Intrinsic::x86_monitorx:
5105 case Intrinsic::x86_clzero: {
5106 bool Use64BitPtr =
Node->getOperand(2).getValueType() == MVT::i64;
5111 case Intrinsic::x86_sse3_monitor:
5112 if (!Subtarget->hasSSE3())
5114 Opc = Use64BitPtr ? X86::MONITOR64rrr : X86::MONITOR32rrr;
5116 case Intrinsic::x86_monitorx:
5117 if (!Subtarget->hasMWAITX())
5119 Opc = Use64BitPtr ? X86::MONITORX64rrr : X86::MONITORX32rrr;
5121 case Intrinsic::x86_clzero:
5122 if (!Subtarget->hasCLZERO())
5124 Opc = Use64BitPtr ? X86::CLZERO64r : X86::CLZERO32r;
5129 unsigned PtrReg = Use64BitPtr ? X86::RAX : X86::EAX;
5130 SDValue Chain = CurDAG->getCopyToReg(
Node->getOperand(0), dl, PtrReg,
5134 if (IntNo == Intrinsic::x86_sse3_monitor ||
5135 IntNo == Intrinsic::x86_monitorx) {
5137 Chain = CurDAG->getCopyToReg(Chain, dl, X86::ECX,
Node->getOperand(3),
5140 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EDX,
Node->getOperand(4),
5145 MachineSDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
5147 ReplaceNode(
Node, CNode);
5153 case Intrinsic::x86_tilestored64_internal: {
5154 unsigned Opc = X86::PTILESTOREDV;
5157 SDValue Scale = getI8Imm(1, dl);
5159 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5160 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5164 Node->getOperand(3),
5170 Node->getOperand(6),
5172 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5173 ReplaceNode(
Node, CNode);
5176 case Intrinsic::x86_tileloadd64:
5177 case Intrinsic::x86_tileloaddt164:
5178 case Intrinsic::x86_tilestored64: {
5179 if (!Subtarget->hasAMXTILE())
5184 case Intrinsic::x86_tileloadd64: Opc = X86::PTILELOADD;
break;
5185 case Intrinsic::x86_tileloaddt164: Opc = X86::PTILELOADDT1;
break;
5186 case Intrinsic::x86_tilestored64: Opc = X86::PTILESTORED;
break;
5189 unsigned TIndex =
Node->getConstantOperandVal(2);
5190 SDValue TReg = getI8Imm(TIndex, dl);
5192 SDValue Scale = getI8Imm(1, dl);
5194 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5195 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5198 if (Opc == X86::PTILESTORED) {
5200 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5203 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5205 ReplaceNode(
Node, CNode);
5213 if (Subtarget->isTargetNaCl())
5217 if (Subtarget->isTarget64BitILP32()) {
5222 assert(
Target.getValueType() == MVT::i32 &&
"Unexpected VT!");
5223 SDValue ZextTarget = CurDAG->getZExtOrTrunc(
Target, dl, MVT::i64);
5224 SDValue Brind = CurDAG->getNode(Opcode, dl, MVT::Other,
5225 Node->getOperand(0), ZextTarget);
5227 SelectCode(ZextTarget.
getNode());
5234 ReplaceNode(
Node, getGlobalBaseReg());
5242 CurDAG->RemoveDeadNode(
Node);
5248 if (matchBitExtract(
Node))
5253 if (tryShiftAmountMod(
Node))
5258 uint8_t
Imm =
Node->getConstantOperandVal(3);
5260 Node->getOperand(1),
Node->getOperand(2), Imm))
5266 if (tryVPTERNLOG(
Node))
5276 tryVPTESTM(
Node, N0, N1))
5279 tryVPTESTM(
Node, N1, N0))
5285 CurDAG->RemoveDeadNode(
Node);
5288 if (matchBitExtract(
Node))
5296 if (tryShrinkShlLogicImm(
Node))
5300 if (tryVPTERNLOG(
Node))
5315 if (!CurDAG->shouldOptForSize())
5319 if (NVT != MVT::i8 && NVT != MVT::i16 && NVT != MVT::i32 && NVT != MVT::i64)
5325 auto *Cst = dyn_cast<ConstantSDNode>(N1);
5329 int64_t Val = Cst->getSExtValue();
5333 if (!isInt<8>(Val) && !isInt<32>(Val))
5337 if (Opcode ==
ISD::ADD && (Val == 1 || Val == -1))
5341 if (!shouldAvoidImmediateInstFormsForSize(N1.
getNode()))
5345 unsigned ROpc, MOpc;
5454 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5455 if (tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5457 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5458 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5462 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N0)->getMemOperand()});
5464 CurDAG->RemoveDeadNode(
Node);
5469 CurDAG->SelectNodeTo(
Node, ROpc, NVT, MVT::i32, N0, N1);
5482 unsigned LoReg, ROpc, MOpc;
5487 ROpc = Opcode ==
X86ISD::SMUL ? X86::IMUL8r : X86::MUL8r;
5488 MOpc = Opcode ==
X86ISD::SMUL ? X86::IMUL8m : X86::MUL8m;
5507 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5508 bool FoldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5511 FoldedLoad = tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5516 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5525 VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5527 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32, MVT::Other);
5531 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5536 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5542 VTs = CurDAG->getVTList(NVT, MVT::i32);
5544 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32);
5546 CNode = CurDAG->getMachineNode(ROpc, dl, VTs, {N1, InGlue});
5551 CurDAG->RemoveDeadNode(
Node);
5561 unsigned LoReg, HiReg;
5563 bool UseMULX = !IsSigned && Subtarget->hasBMI2();
5568 Opc = UseMULXHi ? X86::MULX32Hrr
5570 : IsSigned ?
X86::IMUL32r
5572 MOpc = UseMULXHi ? X86::MULX32Hrm
5574 : IsSigned ?
X86::IMUL32m
5576 LoReg = UseMULX ? X86::EDX : X86::EAX;
5580 Opc = UseMULXHi ? X86::MULX64Hrr
5582 : IsSigned ?
X86::IMUL64r
5584 MOpc = UseMULXHi ? X86::MULX64Hrm
5586 : IsSigned ?
X86::IMUL64m
5588 LoReg = UseMULX ? X86::RDX : X86::RAX;
5593 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5594 bool foldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5597 foldedLoad = tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5602 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5611 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
5612 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5615 }
else if (UseMULX) {
5616 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other);
5617 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5622 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5623 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5629 ReplaceUses(N1.
getValue(1), Chain);
5631 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5633 SDValue Ops[] = { N1, InGlue };
5635 SDVTList VTs = CurDAG->getVTList(NVT);
5636 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5638 }
else if (UseMULX) {
5639 SDVTList VTs = CurDAG->getVTList(NVT, NVT);
5640 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5644 SDVTList VTs = CurDAG->getVTList(MVT::Glue);
5645 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5653 assert(LoReg &&
"Register for low half is not defined!");
5654 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg,
5665 assert(HiReg &&
"Register for high half is not defined!");
5666 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg,
5675 CurDAG->RemoveDeadNode(
Node);
5684 unsigned ROpc, MOpc;
5689 case MVT::i8: ROpc = X86::DIV8r; MOpc = X86::DIV8m;
break;
5690 case MVT::i16: ROpc = X86::DIV16r; MOpc = X86::DIV16m;
break;
5691 case MVT::i32: ROpc = X86::DIV32r; MOpc = X86::DIV32m;
break;
5692 case MVT::i64: ROpc = X86::DIV64r; MOpc = X86::DIV64m;
break;
5697 case MVT::i8: ROpc = X86::IDIV8r; MOpc = X86::IDIV8m;
break;
5698 case MVT::i16: ROpc = X86::IDIV16r; MOpc = X86::IDIV16m;
break;
5699 case MVT::i32: ROpc = X86::IDIV32r; MOpc = X86::IDIV32m;
break;
5700 case MVT::i64: ROpc = X86::IDIV64r; MOpc = X86::IDIV64m;
break;
5704 unsigned LoReg, HiReg, ClrReg;
5705 unsigned SExtOpcode;
5709 LoReg = X86::AL; ClrReg = HiReg = X86::AH;
5713 LoReg = X86::AX; HiReg = X86::DX;
5715 SExtOpcode = X86::CWD;
5718 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX;
5719 SExtOpcode = X86::CDQ;
5722 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX;
5723 SExtOpcode = X86::CQO;
5727 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5728 bool foldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5729 bool signBitIsZero = CurDAG->SignBitIsZero(N0);
5732 if (NVT == MVT::i8) {
5735 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain;
5737 if (tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5739 unsigned Opc = (
isSigned && !signBitIsZero) ? X86::MOVSX16rm8
5741 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, MVT::Other, Ops);
5743 ReplaceUses(N0.
getValue(1), Chain);
5745 CurDAG->setNodeMemRefs(Move, {cast<LoadSDNode>(N0)->getMemOperand()});
5747 unsigned Opc = (
isSigned && !signBitIsZero) ? X86::MOVSX16rr8
5749 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, N0);
5750 Chain = CurDAG->getEntryNode();
5752 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX,
SDValue(Move, 0),
5757 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
5758 LoReg, N0,
SDValue()).getValue(1);
5762 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InGlue),0);
5765 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
5767 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
5771 SDValue(CurDAG->getMachineNode(
5772 TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode,
5773 CurDAG->getTargetConstant(X86::sub_16bit, dl,
5781 SDValue(CurDAG->getMachineNode(
5782 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
5783 CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode,
5784 CurDAG->getTargetConstant(X86::sub_32bit, dl,
5792 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg,
5793 ClrNode, InGlue).getValue(1);
5801 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops);
5806 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5809 SDValue(CurDAG->getMachineNode(ROpc, dl, MVT::Glue, N1, InGlue), 0);
5819 if (HiReg == X86::AH && !
SDValue(
Node, 1).use_empty()) {
5820 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8);
5821 unsigned AHExtOpcode =
5822 isSigned ? X86::MOVSX32rr8_NOREX : X86::MOVZX32rr8_NOREX;
5824 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32,
5825 MVT::Glue, AHCopy, InGlue);
5830 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result);
5838 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5839 LoReg, NVT, InGlue);
5840 InGlue =
Result.getValue(2);
5847 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5848 HiReg, NVT, InGlue);
5849 InGlue =
Result.getValue(2);
5854 CurDAG->RemoveDeadNode(
Node);
5863 SDValue N0 =
Node->getOperand(IsStrictCmp ? 1 : 0);
5864 SDValue N1 =
Node->getOperand(IsStrictCmp ? 2 : 1);
5870 if (Subtarget->canUseCMOV())
5879 Opc = IsSignaling ? X86::COM_Fpr32 : X86::UCOM_Fpr32;
5882 Opc = IsSignaling ? X86::COM_Fpr64 : X86::UCOM_Fpr64;
5885 Opc = IsSignaling ? X86::COM_Fpr80 : X86::UCOM_Fpr80;
5890 IsStrictCmp ?
Node->getOperand(0) : CurDAG->getEntryNode();
5893 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5894 Chain =
SDValue(CurDAG->getMachineNode(Opc, dl, VTs, {N0, N1, Chain}), 0);
5897 Glue =
SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N0, N1), 0);
5902 SDValue(CurDAG->getMachineNode(X86::FNSTSW16r, dl, MVT::i16, Glue), 0);
5906 CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, MVT::i8, FNSTSW);
5910 assert(Subtarget->canUseLAHFSAHF() &&
5911 "Target doesn't support SAHF or FCOMI?");
5912 SDValue AH = CurDAG->getCopyToReg(Chain, dl, X86::AH, Extract,
SDValue());
5915 CurDAG->getMachineNode(X86::SAHF, dl, MVT::i32, AH.
getValue(1)), 0);
5921 CurDAG->RemoveDeadNode(
Node);
5942 unsigned TestOpc = CmpVT == MVT::i64 ? X86::TEST64rr
5945 NewNode = CurDAG->getMachineNode(TestOpc, dl, MVT::i32, BEXTR, BEXTR);
5947 CurDAG->RemoveDeadNode(
Node);
5961 auto *MaskC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
5979 unsigned TestOpcode;
5987 if (LeadingZeros == 0 && SavesBytes) {
5992 ShiftAmt = TrailingZeros;
5994 TestOpcode = X86::TEST64rr;
5995 }
else if (TrailingZeros == 0 && SavesBytes) {
6000 ShiftAmt = LeadingZeros;
6002 TestOpcode = X86::TEST64rr;
6003 }
else if (MaskC->hasOneUse() && !isInt<32>(Mask)) {
6006 unsigned PopCount = 64 - LeadingZeros - TrailingZeros;
6007 if (PopCount == 8) {
6009 ShiftAmt = TrailingZeros;
6010 SubRegIdx = X86::sub_8bit;
6012 TestOpcode = X86::TEST8rr;
6013 }
else if (PopCount == 16) {
6015 ShiftAmt = TrailingZeros;
6016 SubRegIdx = X86::sub_16bit;
6017 SubRegVT = MVT::i16;
6018 TestOpcode = X86::TEST16rr;
6019 }
else if (PopCount == 32) {
6021 ShiftAmt = TrailingZeros;
6022 SubRegIdx = X86::sub_32bit;
6023 SubRegVT = MVT::i32;
6024 TestOpcode = X86::TEST32rr;
6028 SDValue ShiftC = CurDAG->getTargetConstant(ShiftAmt, dl, MVT::i64);
6030 CurDAG->getMachineNode(ShiftOpcode, dl, MVT::i64, MVT::i32,
6033 if (SubRegIdx != 0) {
6035 CurDAG->getTargetExtractSubreg(SubRegIdx, dl, SubRegVT, Shift);
6038 CurDAG->getMachineNode(TestOpcode, dl, MVT::i32, Shift, Shift);
6046 unsigned ROpc, MOpc;
6053 if (isUInt<8>(Mask) &&
6054 (!(Mask & 0x80) || CmpVT == MVT::i8 ||
6058 SubRegOp = X86::sub_8bit;
6059 ROpc = X86::TEST8ri;
6060 MOpc = X86::TEST8mi;
6061 }
else if (OptForMinSize && isUInt<16>(Mask) &&
6062 (!(Mask & 0x8000) || CmpVT == MVT::i16 ||
6069 SubRegOp = X86::sub_16bit;
6070 ROpc = X86::TEST16ri;
6071 MOpc = X86::TEST16mi;
6072 }
else if (isUInt<32>(Mask) && N0.
getValueType() != MVT::i16 &&
6073 ((!(Mask & 0x80000000) &&
6076 (CmpVT != MVT::i16 || !(Mask & 0x8000))) ||
6077 CmpVT == MVT::i32 ||
6085 SubRegOp = X86::sub_32bit;
6086 ROpc = X86::TEST32ri;
6087 MOpc = X86::TEST32mi;
6093 SDValue Imm = CurDAG->getTargetConstant(Mask, dl, VT);
6098 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
6099 if (tryFoldLoad(
Node, N0.
getNode(), Reg, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
6101 if (!LoadN->isSimple()) {
6103 if ((MOpc == X86::TEST8mi && NumVolBits != 8) ||
6104 (MOpc == X86::TEST16mi && NumVolBits != 16) ||
6105 (MOpc == X86::TEST32mi && NumVolBits != 32))
6109 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
6110 Reg.getOperand(0) };
6111 NewNode = CurDAG->getMachineNode(MOpc, dl, MVT::i32, MVT::Other, Ops);
6113 ReplaceUses(
Reg.getValue(1),
SDValue(NewNode, 1));
6115 CurDAG->setNodeMemRefs(NewNode,
6116 {cast<LoadSDNode>(Reg)->getMemOperand()});
6120 Reg = CurDAG->getTargetExtractSubreg(SubRegOp, dl, VT, Reg);
6122 NewNode = CurDAG->getMachineNode(ROpc, dl, MVT::i32, Reg, Imm);
6125 ReplaceNode(
Node, NewNode);
6131 if (!Subtarget->hasSSE42())
6137 bool MayFoldLoad = !NeedIndex || !NeedMask;
6142 Subtarget->hasAVX() ? X86::VPCMPISTRMrri : X86::PCMPISTRMrri;
6144 Subtarget->hasAVX() ? X86::VPCMPISTRMrmi : X86::PCMPISTRMrmi;
6145 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8,
Node);
6148 if (NeedIndex || !NeedMask) {
6150 Subtarget->hasAVX() ? X86::VPCMPISTRIrri : X86::PCMPISTRIrri;
6152 Subtarget->hasAVX() ? X86::VPCMPISTRIrmi : X86::PCMPISTRIrmi;
6153 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32,
Node);
6159 CurDAG->RemoveDeadNode(
Node);
6163 if (!Subtarget->hasSSE42())
6167 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX,
6168 Node->getOperand(1),
6170 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EDX,
6171 Node->getOperand(3), InGlue).getValue(1);
6176 bool MayFoldLoad = !NeedIndex || !NeedMask;
6181 Subtarget->hasAVX() ? X86::VPCMPESTRMrri : X86::PCMPESTRMrri;
6183 Subtarget->hasAVX() ? X86::VPCMPESTRMrmi : X86::PCMPESTRMrmi;
6185 emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8,
Node, InGlue);
6188 if (NeedIndex || !NeedMask) {
6190 Subtarget->hasAVX() ? X86::VPCMPESTRIrri : X86::PCMPESTRIrri;
6192 Subtarget->hasAVX() ? X86::VPCMPESTRIrmi : X86::PCMPESTRIrmi;
6193 CNode = emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32,
Node, InGlue);
6198 CurDAG->RemoveDeadNode(
Node);
6210 if (foldLoadStoreIntoMemOperand(
Node))
6215 MVT VT =
Node->getSimpleValueType(0);
6217 if (Subtarget->hasSBBDepBreaking()) {
6222 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
6227 unsigned Opc = VT == MVT::i64 ? X86::SETB_C64r : X86::SETB_C32r;
6228 MVT SetVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
6230 CurDAG->getMachineNode(Opc, dl, SetVT, EFLAGS, EFLAGS.
getValue(1)),
6239 if (VT == MVT::i8 || VT == MVT::i16) {
6240 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6241 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6245 CurDAG->RemoveDeadNode(
Node);
6259 MVT VT =
Node->getSimpleValueType(0);
6260 if (VT == MVT::i8 || VT == MVT::i16) {
6261 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6262 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6267 CurDAG->RemoveDeadNode(
Node);
6273 auto *Mgt = cast<X86MaskedGatherSDNode>(
Node);
6274 SDValue IndexOp = Mgt->getIndex();
6277 MVT ValueVT =
Node->getSimpleValueType(0);
6278 MVT MaskVT =
Mask.getSimpleValueType();
6295 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6296 Opc = IsFP ? X86::VGATHERDPSZ128rm : X86::VPGATHERDDZ128rm;
6297 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6298 Opc = IsFP ? X86::VGATHERDPSZ256rm : X86::VPGATHERDDZ256rm;
6299 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6300 Opc = IsFP ? X86::VGATHERDPSZrm : X86::VPGATHERDDZrm;
6301 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6302 Opc = IsFP ? X86::VGATHERDPDZ128rm : X86::VPGATHERDQZ128rm;
6303 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6304 Opc = IsFP ? X86::VGATHERDPDZ256rm : X86::VPGATHERDQZ256rm;
6305 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6306 Opc = IsFP ? X86::VGATHERDPDZrm : X86::VPGATHERDQZrm;
6307 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6308 Opc = IsFP ? X86::VGATHERQPSZ128rm : X86::VPGATHERQDZ128rm;
6309 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6310 Opc = IsFP ? X86::VGATHERQPSZ256rm : X86::VPGATHERQDZ256rm;
6311 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6312 Opc = IsFP ? X86::VGATHERQPSZrm : X86::VPGATHERQDZrm;
6313 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6314 Opc = IsFP ? X86::VGATHERQPDZ128rm : X86::VPGATHERQQZ128rm;
6315 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6316 Opc = IsFP ? X86::VGATHERQPDZ256rm : X86::VPGATHERQQZ256rm;
6317 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6318 Opc = IsFP ? X86::VGATHERQPDZrm : X86::VPGATHERQQZrm;
6320 assert(
EVT(MaskVT) ==
EVT(ValueVT).changeVectorElementTypeToInteger() &&
6321 "Unexpected mask VT!");
6322 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6323 Opc = IsFP ? X86::VGATHERDPSrm : X86::VPGATHERDDrm;
6324 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6325 Opc = IsFP ? X86::VGATHERDPSYrm : X86::VPGATHERDDYrm;
6326 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6327 Opc = IsFP ? X86::VGATHERDPDrm : X86::VPGATHERDQrm;
6328 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6329 Opc = IsFP ? X86::VGATHERDPDYrm : X86::VPGATHERDQYrm;
6330 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6331 Opc = IsFP ? X86::VGATHERQPSrm : X86::VPGATHERQDrm;
6332 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6333 Opc = IsFP ? X86::VGATHERQPSYrm : X86::VPGATHERQDYrm;
6334 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6335 Opc = IsFP ? X86::VGATHERQPDrm : X86::VPGATHERQQrm;
6336 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6337 Opc = IsFP ? X86::VGATHERQPDYrm : X86::VPGATHERQQYrm;
6344 if (!selectVectorAddr(Mgt, Mgt->getBasePtr(), IndexOp, Mgt->getScale(),
6348 SDValue PassThru = Mgt->getPassThru();
6349 SDValue Chain = Mgt->getChain();
6351 SDVTList VTs = CurDAG->getVTList(ValueVT, MaskVT, MVT::Other);
6356 Index, Disp, Segment, Chain};
6357 NewNode = CurDAG->getMachineNode(Opc,
SDLoc(dl), VTs, Ops);
6360 Disp, Segment,
Mask, Chain};
6361 NewNode = CurDAG->getMachineNode(Opc,
SDLoc(dl), VTs, Ops);
6363 CurDAG->setNodeMemRefs(NewNode, {Mgt->getMemOperand()});
6366 CurDAG->RemoveDeadNode(
Node);
6370 auto *Sc = cast<X86MaskedScatterSDNode>(
Node);
6372 SDValue IndexOp = Sc->getIndex();
6374 MVT ValueVT =
Value.getSimpleValueType();
6389 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6390 Opc = IsFP ? X86::VSCATTERDPSZ128mr : X86::VPSCATTERDDZ128mr;
6391 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6392 Opc = IsFP ? X86::VSCATTERDPSZ256mr : X86::VPSCATTERDDZ256mr;
6393 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6394 Opc = IsFP ? X86::VSCATTERDPSZmr : X86::VPSCATTERDDZmr;
6395 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6396 Opc = IsFP ? X86::VSCATTERDPDZ128mr : X86::VPSCATTERDQZ128mr;
6397 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6398 Opc = IsFP ? X86::VSCATTERDPDZ256mr : X86::VPSCATTERDQZ256mr;
6399 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6400 Opc = IsFP ? X86::VSCATTERDPDZmr : X86::VPSCATTERDQZmr;
6401 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6402 Opc = IsFP ? X86::VSCATTERQPSZ128mr : X86::VPSCATTERQDZ128mr;
6403 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6404 Opc = IsFP ? X86::VSCATTERQPSZ256mr : X86::VPSCATTERQDZ256mr;
6405 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6406 Opc = IsFP ? X86::VSCATTERQPSZmr : X86::VPSCATTERQDZmr;
6407 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6408 Opc = IsFP ? X86::VSCATTERQPDZ128mr : X86::VPSCATTERQQZ128mr;
6409 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6410 Opc = IsFP ? X86::VSCATTERQPDZ256mr : X86::VPSCATTERQQZ256mr;
6411 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6412 Opc = IsFP ? X86::VSCATTERQPDZmr : X86::VPSCATTERQQZmr;
6417 if (!selectVectorAddr(Sc, Sc->getBasePtr(), IndexOp, Sc->getScale(),
6422 SDValue Chain = Sc->getChain();
6424 SDVTList VTs = CurDAG->getVTList(
Mask.getValueType(), MVT::Other);
6428 CurDAG->setNodeMemRefs(NewNode, {Sc->getMemOperand()});
6430 CurDAG->RemoveDeadNode(
Node);
6436 cast<SrcValueSDNode>(
Node->getOperand(1))->getValue());
6438 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6440 TargetOpcode::PREALLOCATED_SETUP, dl, MVT::Other, CallIdValue, Chain);
6442 CurDAG->RemoveDeadNode(
Node);
6448 cast<SrcValueSDNode>(
Node->getOperand(1))->getValue());
6450 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6453 Ops[0] = CallIdValue;
6457 TargetOpcode::PREALLOCATED_ARG, dl,
6458 CurDAG->getVTList(TLI->
getPointerTy(CurDAG->getDataLayout()),
6463 CurDAG->RemoveDeadNode(
Node);
6470 if (!Subtarget->hasWIDEKL())
6474 switch (
Node->getOpcode()) {
6478 Opcode = X86::AESENCWIDE128KL;
6481 Opcode = X86::AESDECWIDE128KL;
6484 Opcode = X86::AESENCWIDE256KL;
6487 Opcode = X86::AESDECWIDE256KL;
6498 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(2),
6500 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(3),
6502 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM2,
Node->getOperand(4),
6504 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM3,
Node->getOperand(5),
6506 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM4,
Node->getOperand(6),
6508 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM5,
Node->getOperand(7),
6510 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM6,
Node->getOperand(8),
6512 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM7,
Node->getOperand(9),
6516 Opcode, dl,
Node->getVTList(),
6517 {Base, Scale, Index, Disp, Segment, Chain, Chain.getValue(1)});
6518 CurDAG->setNodeMemRefs(Res, cast<MemSDNode>(
Node)->getMemOperand());
6519 ReplaceNode(
Node, Res);
6527bool X86DAGToDAGISel::SelectInlineAsmMemoryOperand(
6529 std::vector<SDValue> &OutOps) {
6530 SDValue Op0, Op1, Op2, Op3, Op4;
6531 switch (ConstraintID) {
6534 case InlineAsm::ConstraintCode::o:
6535 case InlineAsm::ConstraintCode::v:
6536 case InlineAsm::ConstraintCode::m:
6537 case InlineAsm::ConstraintCode::X:
6538 case InlineAsm::ConstraintCode::p:
6539 if (!selectAddr(
nullptr,
Op, Op0, Op1, Op2, Op3, Op4))
6544 OutOps.push_back(Op0);
6545 OutOps.push_back(Op1);
6546 OutOps.push_back(Op2);
6547 OutOps.push_back(Op3);
6548 OutOps.push_back(Op4);
6556 return new X86DAGToDAGISel(
TM, OptLevel);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool isRIPRelative(const MCInst &MI, const MCInstrInfo &MCII)
Check if the instruction uses RIP relative addressing.
static bool isLegalMaskCompare(SDNode *N, const X86Subtarget *Subtarget)
static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, unsigned LoadOpNo, LoadSDNode *&LoadNode, SDValue &InputChain)
Check whether or not the chain ending in StoreNode is suitable for doing the {load; op; store} to mod...
#define GET_EGPR_IF_ENABLED(OPC)
static bool needBWI(MVT VT)
static unsigned getVPTESTMOpc(MVT TestVT, bool IsTestN, bool FoldedLoad, bool FoldedBCast, bool Masked)
static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM, const X86Subtarget &Subtarget)
static bool mayUseCarryFlag(X86::CondCode CC)
static cl::opt< bool > EnablePromoteAnyextLoad("x86-promote-anyext-load", cl::init(true), cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden)
cl::opt< bool > IndirectBranchTracking
static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, SDValue Call, SDValue OrigChain)
Replace the original chain operand of the call with load's chain operand and move load below the call...
static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N)
#define GET_ND_IF_ENABLED(OPC)
#define VPTESTM_BROADCAST_CASES(SUFFIX)
static cl::opt< bool > AndImmShrink("x86-and-imm-shrink", cl::init(true), cl::desc("Enable setting constant bits to reduce size of mask immediates"), cl::Hidden)
static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, X86ISelAddressMode &AM)
#define VPTESTM_FULL_CASES(SUFFIX)
static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq)
Return true if call address is a load and it can be moved below CALLSEQ_START and the chains leading ...
static bool isDispSafeForFrameIndex(int64_t Val)
static bool isEndbrImm64(uint64_t Imm)
#define GET_ND_IF_ENABLED(OPC)
DEMANGLE_DUMP_METHOD void dump() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
APInt trunc(unsigned width) const
Truncate to new width.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countl_zero() const
The APInt version of std::countl_zero.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
bool isOne() const
Determine if this is a value of 1.
unsigned countr_one() const
Count the number of trailing one bits.
The address of a basic block.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
std::optional< ConstantRange > getAbsoluteSymbolRange() const
If this is an absolute symbol reference, returns the range of the symbol, otherwise returns std::null...
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
Describe properties that are true of each instruction in the target description file.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
bool is128BitVector() const
Return true if this is a 128-bit vector type.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool is512BitVector() const
Return true if this is a 512-bit vector type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const Module * getModule() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isNonTemporal() const
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
static int getUninvalidatedNodeId(SDNode *N)
virtual void emitFunctionEntryCode()
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
static constexpr unsigned MaxRecursionDepth
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
static Type * getVoidTy(LLVMContext &C)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void dump() const
Support for debugging, callable in GDB: V->dump()
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
size_t getPreallocatedIdForCallSite(const Value *CS)
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BRIND
BRIND - Indirect branch.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
@ GlobalBaseReg
The result of the mflr at function entry, used for PIC code.
@ X86
Windows x64, Windows Itanium (IA-64)
Reg
All possible values of the reg field in the ModR/M byte.
@ MO_NO_FLAG
MO_NO_FLAG - No flag for the operand.
@ EVEX
EVEX - Specifies that this instruction use EVEX form which provides syntax support up to 32 512-bit r...
@ VEX
VEX - encoding using 0xC4/0xC5.
@ XOP
XOP - Opcode prefix used by XOP instructions.
@ FST
This instruction implements a truncating store from FP stack slots.
@ CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ CMP
X86 compare and logical compare instructions.
@ BLENDV
Dynamic (non-constant condition) vector blend where only the sign bits of the condition elements are ...
@ STRICT_FCMP
X86 strict FP compare instructions.
@ STRICT_CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ NT_BRIND
BRIND node with NoTrack prefix.
@ FSETCCM
X86 FP SETCC, similar to above, but with output as an i1 mask and and a version with SAE.
@ FXOR
Bitwise logical XOR of floating point values.
@ BRCOND
X86 conditional branches.
@ CALL
These operations represent an abstract X86 call instruction, which includes a bunch of information.
@ FANDN
Bitwise logical ANDNOT of floating point values.
@ GlobalBaseReg
On Darwin, this node represents the result of the popl at function entry, used for PIC code.
@ FLD
This instruction implements an extending load to FP stack slots.
@ TC_RETURN
Tail call return.
@ FOR
Bitwise logical OR of floating point values.
@ Wrapper
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress,...
@ ANDNP
Bitwise Logical AND NOT of Packed FP values.
@ FAND
Bitwise logical AND of floating point values.
@ CMOV
X86 conditional moves.
@ WrapperRIP
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
int getCondSrcNoFromDesc(const MCInstrDesc &MCID)
Return the source operand # for condition code by MCID.
bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into some other x86 instruction as a memory oper...
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement)
Returns true of the given offset can be fit into displacement field of the instruction.
bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs)
If Op is a constant whose elements are all the same constant or undefined, return true and return the...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
int popcount(T Value) noexcept
Count the number of set bits in a value.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
unsigned M1(unsigned Val)
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
FunctionPass * createX86ISelDag(X86TargetMachine &TM, CodeGenOptLevel OptLevel)
This pass converts a legalized DAG into a X86-specific DAG, ready for instruction scheduling.
CodeGenOptLevel
Code generation optimization level.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.