22#include "llvm/Config/llvm-config.h"
27#include "llvm/IR/IntrinsicsX86.h"
37#define DEBUG_TYPE "x86-isel"
38#define PASS_NAME "X86 DAG->DAG Instruction Selection"
40STATISTIC(NumLoadMoved,
"Number of loads moved below TokenFactor");
43 cl::desc(
"Enable setting constant bits to reduce size of mask immediates"),
47 "x86-promote-anyext-load",
cl::init(
true),
59 struct X86ISelAddressMode {
67 int Base_FrameIndex = 0;
76 const char *ES =
nullptr;
81 bool NegateIndex =
false;
83 X86ISelAddressMode() =
default;
85 bool hasSymbolicDisplacement()
const {
86 return GV !=
nullptr ||
CP !=
nullptr || ES !=
nullptr ||
87 MCSym !=
nullptr ||
JT != -1 || BlockAddr !=
nullptr;
90 bool hasBaseOrIndexReg()
const {
97 if (
BaseType != RegBase)
return false;
99 dyn_cast_or_null<RegisterSDNode>(Base_Reg.
getNode()))
100 return RegNode->getReg() == X86::RIP;
109#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
111 dbgs() <<
"X86ISelAddressMode " <<
this <<
'\n';
112 dbgs() <<
"Base_Reg ";
118 dbgs() <<
" Base.FrameIndex " << Base_FrameIndex <<
'\n';
119 dbgs() <<
" Scale " << Scale <<
'\n'
127 dbgs() <<
" Disp " << Disp <<
'\n'
149 dbgs() <<
" JT" <<
JT <<
" Align" << Alignment.
value() <<
'\n';
169 bool IndirectTlsSegRefs;
174 X86DAGToDAGISel() =
delete;
178 OptForMinSize(
false), IndirectTlsSegRefs(
false) {}
184 "indirect-tls-seg-refs");
189 "OptForMinSize implies OptForSize");
203#include "X86GenDAGISel.inc"
208 bool foldOffsetIntoAddress(
uint64_t Offset, X86ISelAddressMode &AM);
209 bool matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
210 bool AllowSegmentRegForX32 =
false);
211 bool matchWrapper(
SDValue N, X86ISelAddressMode &AM);
212 bool matchAddress(
SDValue N, X86ISelAddressMode &AM);
213 bool matchVectorAddress(
SDValue N, X86ISelAddressMode &AM);
214 bool matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
unsigned Depth);
217 bool matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
219 bool matchVectorAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
221 bool matchAddressBase(
SDValue N, X86ISelAddressMode &AM);
250 return tryFoldLoad(
P,
P,
N,
Base, Scale,
Index, Disp, Segment);
258 bool isProfitableToFormMaskedOp(
SDNode *
N)
const;
263 std::vector<SDValue> &OutOps)
override;
265 void emitSpecialCodeForMain();
267 inline void getAddressOperands(X86ISelAddressMode &AM,
const SDLoc &
DL,
271 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
272 Base = CurDAG->getTargetFrameIndex(
273 AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout()));
274 else if (AM.Base_Reg.getNode())
277 Base = CurDAG->getRegister(0, VT);
279 Scale = getI8Imm(AM.Scale,
DL);
281#define GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC)
283 if (AM.NegateIndex) {
291 if (AM.IndexReg.getNode())
294 Index = CurDAG->getRegister(0, VT);
299 Disp = CurDAG->getTargetGlobalAddress(AM.GV,
SDLoc(),
303 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment,
304 AM.Disp, AM.SymbolFlags);
306 assert(!AM.Disp &&
"Non-zero displacement is ignored with ES.");
307 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
308 }
else if (AM.MCSym) {
309 assert(!AM.Disp &&
"Non-zero displacement is ignored with MCSym.");
310 assert(AM.SymbolFlags == 0 &&
"oo");
311 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32);
312 }
else if (AM.JT != -1) {
313 assert(!AM.Disp &&
"Non-zero displacement is ignored with JT.");
314 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
315 }
else if (AM.BlockAddr)
316 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
319 Disp = CurDAG->getTargetConstant(AM.Disp,
DL, MVT::i32);
321 if (AM.Segment.getNode())
322 Segment = AM.Segment;
324 Segment = CurDAG->getRegister(0, MVT::i16);
333 bool shouldAvoidImmediateInstFormsForSize(
SDNode *
N)
const {
339 if (!CurDAG->shouldOptForSize())
349 if (
User->isMachineOpcode()) {
372 auto *
C = dyn_cast<ConstantSDNode>(
N);
373 if (
C && isInt<8>(
C->getSExtValue()))
393 (RegNode = dyn_cast_or_null<RegisterSDNode>(
395 if ((RegNode->
getReg() == X86::ESP) ||
396 (RegNode->
getReg() == X86::RSP))
405 return (UseCount > 1);
410 return CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
415 return CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
420 return CurDAG->getTargetConstant(Imm,
DL, MVT::i64);
425 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
427 MVT VecVT =
N->getOperand(0).getSimpleValueType();
433 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
435 MVT VecVT =
N->getSimpleValueType(0);
439 SDValue getPermuteVINSERTCommutedImmediate(
SDNode *
N,
unsigned VecWidth,
441 assert(VecWidth == 128 &&
"Unexpected vector width");
443 MVT VecVT =
N->getSimpleValueType(0);
445 assert((InsertIdx == 0 || InsertIdx == 1) &&
"Bad insertf128 index");
448 return getI8Imm(InsertIdx ? 0x02 : 0x30,
DL);
453 MVT VT =
N->getSimpleValueType(0);
456 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
458 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
459 if (VT == MVT::i64) {
461 CurDAG->getMachineNode(
462 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
463 CurDAG->getTargetConstant(0, dl, MVT::i64), Zero,
464 CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)),
469 unsigned Opcode =
N->getOpcode();
471 "Unexpected opcode for SBB materialization");
472 unsigned FlagOpIndex = Opcode ==
X86ISD::SBB ? 2 : 1;
474 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
475 N->getOperand(FlagOpIndex),
SDValue());
479 unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr;
480 MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
481 VTs = CurDAG->getVTList(SBBVT, MVT::i32);
483 CurDAG->getMachineNode(Opc, dl, VTs,
484 {Zero, Zero, EFLAGS, EFLAGS.getValue(1)}),
490 bool isUnneededShiftMask(
SDNode *
N,
unsigned Width)
const {
492 const APInt &Val =
N->getConstantOperandAPInt(1);
497 APInt Mask = Val | CurDAG->computeKnownBits(
N->getOperand(0)).Zero;
498 return Mask.countr_one() >= Width;
504 SDNode *getGlobalBaseReg();
515 return Subtarget->getInstrInfo();
528 bool isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const;
532 if (!
N->isNonTemporal())
535 unsigned StoreSize =
N->getMemoryVT().getStoreSize();
537 if (
N->getAlign().value() < StoreSize)
546 return Subtarget->hasSSE41();
548 return Subtarget->hasAVX2();
550 return Subtarget->hasAVX512();
554 bool foldLoadStoreIntoMemOperand(
SDNode *
Node);
557 bool shrinkAndImmediate(
SDNode *
N);
558 bool isMaskZeroExtended(
SDNode *
N)
const;
559 bool tryShiftAmountMod(
SDNode *
N);
560 bool tryShrinkShlLogicImm(
SDNode *
N);
566 bool tryMatchBitSelect(
SDNode *
N);
568 MachineSDNode *emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
570 MachineSDNode *emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
574 bool tryOptimizeRem8Extend(
SDNode *
N);
576 bool onlyUsesZeroFlag(
SDValue Flags)
const;
577 bool hasNoSignFlagUses(
SDValue Flags)
const;
578 bool hasNoCarryFlagUses(
SDValue Flags)
const;
582char X86DAGToDAGISel::ID = 0;
589 unsigned Opcode =
N->getOpcode();
596 EVT OpVT =
N->getOperand(0).getValueType();
600 OpVT =
N->getOperand(1).getValueType();
602 return Subtarget->hasVLX();
616bool X86DAGToDAGISel::isMaskZeroExtended(
SDNode *
N)
const {
629 if (OptLevel == CodeGenOptLevel::None)
639 if (useNonTemporalLoad(cast<LoadSDNode>(
N)))
644 switch (
U->getOpcode()) {
670 if (
auto *Imm = dyn_cast<ConstantSDNode>(Op1)) {
671 if (
Imm->getAPIntValue().isSignedIntN(8))
680 Imm->getAPIntValue().getBitWidth() == 64 &&
681 Imm->getAPIntValue().isIntN(32))
688 (
Imm->getAPIntValue() == UINT8_MAX ||
689 Imm->getAPIntValue() == UINT16_MAX ||
690 Imm->getAPIntValue() == UINT32_MAX))
696 (-
Imm->getAPIntValue()).isSignedIntN(8))
700 (-
Imm->getAPIntValue()).isSignedIntN(8) &&
701 hasNoCarryFlagUses(
SDValue(U, 1)))
726 if (
U->getOperand(0).getOpcode() ==
ISD::SHL &&
730 if (
U->getOperand(1).getOpcode() ==
ISD::SHL &&
738 auto *
C = dyn_cast<ConstantSDNode>(U0.
getOperand(0));
739 if (
C &&
C->getSExtValue() == -2)
744 auto *
C = dyn_cast<ConstantSDNode>(U1.
getOperand(0));
745 if (
C &&
C->getSExtValue() == -2)
759 if (isa<ConstantSDNode>(
U->getOperand(1)))
780bool X86DAGToDAGISel::isProfitableToFormMaskedOp(
SDNode *
N)
const {
783 "Unexpected opcode!");
788 return N->getOperand(1).hasOneUse();
797 if (Chain.
getNode() == Load.getNode())
801 "Unexpected chain operand");
815 Load.getOperand(1), Load.getOperand(2));
819 Ops.
append(Call->op_begin() + 1, Call->op_end());
833 if (Callee.getNode() == Chain.
getNode() || !Callee.hasOneUse())
835 auto *LD = dyn_cast<LoadSDNode>(Callee.getNode());
853 if (isa<MemSDNode>(Chain.
getNode()) &&
854 cast<MemSDNode>(Chain.
getNode())->writeMem())
860 Callee.getValue(1).hasOneUse())
868 if ((Imm & 0x00FFFFFF) != 0x0F1EFA)
871 uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64,
872 0x65, 0x66, 0x67, 0xf0, 0xf2};
875 uint8_t Byte = (Imm >> i) & 0xFF;
887 return (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v64i8);
890void X86DAGToDAGISel::PreprocessISelDAG() {
891 bool MadeChange =
false;
893 E = CurDAG->allnodes_end();
I != E; ) {
912 MVT VT =
N->getSimpleValueType(0);
913 int64_t
Imm = cast<ConstantSDNode>(
N)->getSExtValue();
914 int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB;
921 SDValue Complement = CurDAG->getConstant(~Imm, dl, VT,
false,
true);
922 Complement = CurDAG->getNOT(dl, Complement, VT);
924 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Complement);
934 if (
N->getOpcode() ==
X86ISD::AND && !
N->hasAnyUseOfValue(1)) {
936 N->getOperand(0),
N->getOperand(1));
938 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
962 auto mayPreventLoadFold = [&]() {
964 N->getOpcode() ==
ISD::ADD && Subtarget->hasAVX() &&
965 !
N->getOperand(1).hasOneUse();
968 N->getSimpleValueType(0).isVector() && !mayPreventLoadFold()) {
974 MVT VT =
N->getSimpleValueType(0);
982 CurDAG->getNode(NewOpcode,
DL, VT,
N->getOperand(0),
AllOnes);
984 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
991 switch (
N->getOpcode()) {
993 MVT VT =
N->getSimpleValueType(0);
995 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1002 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1005 CurDAG->getIntPtrConstant(
Index, dl));
1008 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1017 MVT VT =
N->getSimpleValueType(0);
1019 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1021 auto *MemNode = cast<MemSDNode>(
N);
1023 SDVTList VTs = CurDAG->getVTList(NarrowVT, MVT::Other);
1024 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()};
1025 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode(
1027 MemNode->getMemOperand());
1030 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1033 CurDAG->getIntPtrConstant(
Index, dl));
1037 CurDAG->ReplaceAllUsesWith(
N, To);
1048 auto *Ld = cast<LoadSDNode>(
N);
1049 MVT VT =
N->getSimpleValueType(0);
1057 SDValue Chain = Ld->getChain();
1059 auto *UserLd = dyn_cast<LoadSDNode>(
User);
1060 MVT UserVT =
User->getSimpleValueType(0);
1062 UserLd->getBasePtr() ==
Ptr && UserLd->getChain() == Chain &&
1063 !
User->hasAnyUseOfValue(1) &&
1077 CurDAG->getIntPtrConstant(0, dl));
1078 SDValue Res = CurDAG->getBitcast(VT, Extract);
1082 CurDAG->ReplaceAllUsesWith(
N, To);
1091 EVT EleVT =
N->getOperand(0).getValueType().getVectorElementType();
1092 if (EleVT == MVT::i1)
1095 assert(Subtarget->hasSSE41() &&
"Expected SSE4.1 support!");
1096 assert(
N->getValueType(0).getVectorElementType() != MVT::i16 &&
1097 "We can't replace VSELECT with BLENDV in vXi16!");
1099 if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(
N->getOperand(0)) ==
1102 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
1103 CurDAG->getTargetConstant(0xCA,
SDLoc(
N), MVT::i8));
1106 N->getOperand(0),
N->getOperand(1),
1110 CurDAG->ReplaceAllUsesWith(
N,
R.getNode());
1123 if (!
N->getSimpleValueType(0).isVector())
1127 switch (
N->getOpcode()) {
1137 if (
N->isStrictFPOpcode())
1139 CurDAG->getNode(NewOpc,
SDLoc(
N), {
N->getValueType(0), MVT::Other},
1140 {
N->getOperand(0),
N->getOperand(1)});
1143 CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1146 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1156 if (!
N->getValueType(0).isVector())
1160 switch (
N->getOpcode()) {
1166 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1167 N->getOperand(0),
N->getOperand(1));
1169 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1178 if (!
N->getValueType(0).isVector())
1182 if (
N->getOperand(0).getScalarValueSizeInBits() == 1) {
1184 "Unexpected opcode for mask vector!");
1192 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1195 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1215 switch (
N->getOpcode()) {
1231 bool IsStrict =
N->isStrictFPOpcode();
1235 {
N->getValueType(0), MVT::Other},
1236 {
N->getOperand(0),
N->getOperand(1),
1237 CurDAG->getTargetConstant(Imm, dl, MVT::i32)});
1241 CurDAG->getTargetConstant(Imm, dl, MVT::i32));
1243 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1254 MVT VT =
N->getSimpleValueType(0);
1255 if (VT.
isVector() || VT == MVT::f128)
1258 MVT VecVT = VT == MVT::f64 ? MVT::v2f64
1259 : VT == MVT::f32 ? MVT::v4f32
1269 if (Subtarget->hasSSE2()) {
1274 switch (
N->getOpcode()) {
1281 Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1);
1284 Res = CurDAG->getNode(
N->getOpcode(), dl, VecVT, Op0, Op1);
1287 CurDAG->getIntPtrConstant(0, dl));
1289 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1296 if (OptLevel != CodeGenOptLevel::None &&
1299 !Subtarget->useIndirectThunkCalls() &&
1300 ((
N->getOpcode() ==
X86ISD::CALL && !Subtarget->slowTwoMemOps()) ||
1302 (Subtarget->is64Bit() ||
1303 !getTargetMachine().isPositionIndependent())))) {
1342 switch (
N->getOpcode()) {
1347 MVT SrcVT =
N->getOperand(0).getSimpleValueType();
1348 MVT DstVT =
N->getSimpleValueType(0);
1360 if (SrcIsSSE && DstIsSSE)
1363 if (!SrcIsSSE && !DstIsSSE) {
1368 if (
N->getConstantOperandVal(1))
1376 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1377 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1385 CurDAG->getEntryNode(), dl,
N->getOperand(0), MemTmp, MPI, MemVT);
1387 MemTmp, MPI, MemVT);
1394 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Result);
1403 MVT SrcVT =
N->getOperand(1).getSimpleValueType();
1404 MVT DstVT =
N->getSimpleValueType(0);
1416 if (SrcIsSSE && DstIsSSE)
1419 if (!SrcIsSSE && !DstIsSSE) {
1424 if (
N->getConstantOperandVal(2))
1432 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1433 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1443 SDVTList VTs = CurDAG->getVTList(MVT::Other);
1444 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), MemTmp};
1448 if (
N->getFlags().hasNoFPExcept()) {
1450 Flags.setNoFPExcept(
true);
1451 Store->setFlags(Flags);
1454 assert(SrcVT == MemVT &&
"Unexpected VT!");
1455 Store = CurDAG->getStore(
N->getOperand(0), dl,
N->getOperand(1), MemTmp,
1460 SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other);
1462 Result = CurDAG->getMemIntrinsicNode(
1465 if (
N->getFlags().hasNoFPExcept()) {
1467 Flags.setNoFPExcept(
true);
1471 assert(DstVT == MemVT &&
"Unexpected VT!");
1472 Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI);
1480 CurDAG->ReplaceAllUsesWith(
N,
Result.getNode());
1494 CurDAG->RemoveDeadNodes();
1498bool X86DAGToDAGISel::tryOptimizeRem8Extend(
SDNode *
N) {
1499 unsigned Opc =
N->getMachineOpcode();
1500 if (Opc != X86::MOVZX32rr8 && Opc != X86::MOVSX32rr8 &&
1501 Opc != X86::MOVSX64rr8)
1513 unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX
1514 : X86::MOVSX32rr8_NOREX;
1519 if (Opc == X86::MOVSX64rr8) {
1524 ReplaceUses(
N, Extend);
1533void X86DAGToDAGISel::PostprocessISelDAG() {
1535 if (
TM.getOptLevel() == CodeGenOptLevel::None)
1540 bool MadeChange =
false;
1541 while (Position != CurDAG->allnodes_begin()) {
1544 if (
N->use_empty() || !
N->isMachineOpcode())
1547 if (tryOptimizeRem8Extend(
N)) {
1554 unsigned Opc =
N->getMachineOpcode();
1555 if ((Opc == X86::TEST8rr || Opc == X86::TEST16rr ||
1556 Opc == X86::TEST32rr || Opc == X86::TEST64rr) &&
1557 N->getOperand(0) ==
N->getOperand(1) &&
1558 N->getOperand(0)->hasNUsesOfValue(2,
N->getOperand(0).getResNo()) &&
1559 N->getOperand(0).isMachineOpcode()) {
1561 unsigned N0Opc =
And.getMachineOpcode();
1562 if ((N0Opc == X86::AND8rr || N0Opc == X86::AND16rr ||
1563 N0Opc == X86::AND32rr || N0Opc == X86::AND64rr ||
1564 N0Opc == X86::AND8rr_ND || N0Opc == X86::AND16rr_ND ||
1565 N0Opc == X86::AND32rr_ND || N0Opc == X86::AND64rr_ND) &&
1566 !
And->hasAnyUseOfValue(1)) {
1571 ReplaceUses(
N,
Test);
1575 if ((N0Opc == X86::AND8rm || N0Opc == X86::AND16rm ||
1576 N0Opc == X86::AND32rm || N0Opc == X86::AND64rm ||
1577 N0Opc == X86::AND8rm_ND || N0Opc == X86::AND16rm_ND ||
1578 N0Opc == X86::AND32rm_ND || N0Opc == X86::AND64rm_ND) &&
1579 !
And->hasAnyUseOfValue(1)) {
1581#define CASE_ND(OP) \
1584#define FROM_TO(A, B) \
1585 CASE_ND(A) NewOpc = X86::B; \
1603 And.getOperand(6) };
1605 MVT::i32, MVT::Other, Ops);
1606 CurDAG->setNodeMemRefs(
1607 Test, cast<MachineSDNode>(
And.getNode())->memoperands());
1619 if ((Opc == X86::KORTESTBrr || Opc == X86::KORTESTWrr ||
1620 Opc == X86::KORTESTDrr || Opc == X86::KORTESTQrr) &&
1621 N->getOperand(0) ==
N->getOperand(1) &&
1622 N->isOnlyUserOf(
N->getOperand(0).getNode()) &&
1623 N->getOperand(0).isMachineOpcode() &&
1626 unsigned N0Opc =
And.getMachineOpcode();
1629 if (N0Opc == X86::KANDBrr ||
1630 (N0Opc == X86::KANDWrr && Subtarget->hasDQI()) ||
1631 N0Opc == X86::KANDDrr || N0Opc == X86::KANDQrr) {
1635 case X86::KORTESTBrr: NewOpc = X86::KTESTBrr;
break;
1636 case X86::KORTESTWrr: NewOpc = X86::KTESTWrr;
break;
1637 case X86::KORTESTDrr: NewOpc = X86::KTESTDrr;
break;
1638 case X86::KORTESTQrr: NewOpc = X86::KTESTQrr;
break;
1644 ReplaceUses(
N, KTest);
1651 if (Opc != TargetOpcode::SUBREG_TO_REG)
1654 unsigned SubRegIdx =
N->getConstantOperandVal(2);
1655 if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm)
1666 case X86::VMOVAPDrr:
case X86::VMOVUPDrr:
1667 case X86::VMOVAPSrr:
case X86::VMOVUPSrr:
1668 case X86::VMOVDQArr:
case X86::VMOVDQUrr:
1669 case X86::VMOVAPDYrr:
case X86::VMOVUPDYrr:
1670 case X86::VMOVAPSYrr:
case X86::VMOVUPSYrr:
1671 case X86::VMOVDQAYrr:
case X86::VMOVDQUYrr:
1672 case X86::VMOVAPDZ128rr:
case X86::VMOVUPDZ128rr:
1673 case X86::VMOVAPSZ128rr:
case X86::VMOVUPSZ128rr:
1674 case X86::VMOVDQA32Z128rr:
case X86::VMOVDQU32Z128rr:
1675 case X86::VMOVDQA64Z128rr:
case X86::VMOVDQU64Z128rr:
1676 case X86::VMOVAPDZ256rr:
case X86::VMOVUPDZ256rr:
1677 case X86::VMOVAPSZ256rr:
case X86::VMOVUPSZ256rr:
1678 case X86::VMOVDQA32Z256rr:
case X86::VMOVDQU32Z256rr:
1679 case X86::VMOVDQA64Z256rr:
case X86::VMOVDQU64Z256rr:
1684 if (!
In.isMachineOpcode() ||
1685 In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END)
1690 uint64_t TSFlags = getInstrInfo()->get(
In.getMachineOpcode()).TSFlags;
1698 CurDAG->UpdateNodeOperands(
N,
N->getOperand(0), In,
N->getOperand(2));
1703 CurDAG->RemoveDeadNodes();
1708void X86DAGToDAGISel::emitSpecialCodeForMain() {
1709 if (Subtarget->isTargetCygMing()) {
1711 auto &
DL = CurDAG->getDataLayout();
1714 CLI.setChain(CurDAG->getRoot())
1716 CurDAG->getExternalSymbol(
"__main", TLI->getPointerTy(
DL)),
1720 CurDAG->setRoot(
Result.second);
1724void X86DAGToDAGISel::emitFunctionEntryCode() {
1727 if (
F.hasExternalLinkage() &&
F.getName() ==
"main")
1728 emitSpecialCodeForMain();
1738 return isInt<31>(Val);
1742 X86ISelAddressMode &AM) {
1747 int64_t Val = AM.Disp +
Offset;
1750 if (Val != 0 && (AM.ES || AM.MCSym))
1754 if (Subtarget->is64Bit()) {
1757 AM.hasSymbolicDisplacement()))
1761 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
1780 if (Subtarget->isTarget64BitILP32() && !isUInt<31>(Val) &&
1781 !AM.hasBaseOrIndexReg())
1788bool X86DAGToDAGISel::matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
1789 bool AllowSegmentRegForX32) {
1801 if (
isNullConstant(Address) && AM.Segment.getNode() ==
nullptr &&
1802 !IndirectTlsSegRefs &&
1803 (Subtarget->isTargetGlibc() || Subtarget->isTargetAndroid() ||
1804 Subtarget->isTargetFuchsia())) {
1805 if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32)
1807 switch (
N->getPointerInfo().getAddrSpace()) {
1809 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
1812 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
1825bool X86DAGToDAGISel::matchWrapper(
SDValue N, X86ISelAddressMode &AM) {
1828 if (AM.hasSymbolicDisplacement())
1831 bool IsRIPRelTLS =
false;
1849 if (IsRIPRel && AM.hasBaseOrIndexReg())
1853 X86ISelAddressMode Backup = AM;
1857 if (
auto *
G = dyn_cast<GlobalAddressSDNode>(N0)) {
1858 AM.GV =
G->getGlobal();
1859 AM.SymbolFlags =
G->getTargetFlags();
1861 }
else if (
auto *CP = dyn_cast<ConstantPoolSDNode>(N0)) {
1862 AM.CP =
CP->getConstVal();
1863 AM.Alignment =
CP->getAlign();
1864 AM.SymbolFlags =
CP->getTargetFlags();
1866 }
else if (
auto *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
1867 AM.ES = S->getSymbol();
1868 AM.SymbolFlags = S->getTargetFlags();
1869 }
else if (
auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
1870 AM.MCSym = S->getMCSymbol();
1871 }
else if (
auto *J = dyn_cast<JumpTableSDNode>(N0)) {
1872 AM.JT = J->getIndex();
1873 AM.SymbolFlags = J->getTargetFlags();
1874 }
else if (
auto *BA = dyn_cast<BlockAddressSDNode>(N0)) {
1875 AM.BlockAddr = BA->getBlockAddress();
1876 AM.SymbolFlags = BA->getTargetFlags();
1877 Offset = BA->getOffset();
1882 if (Subtarget->is64Bit() && !IsRIPRel && AM.GV &&
1883 TM.isLargeGlobalValue(AM.GV)) {
1888 if (foldOffsetIntoAddress(
Offset, AM)) {
1894 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64));
1902bool X86DAGToDAGISel::matchAddress(
SDValue N, X86ISelAddressMode &AM) {
1903 if (matchAddressRecursively(
N, AM, 0))
1910 if (Subtarget->isTarget64BitILP32() &&
1911 AM.BaseType == X86ISelAddressMode::RegBase &&
1912 AM.Base_Reg.getNode() !=
nullptr && AM.IndexReg.getNode() ==
nullptr) {
1913 SDValue Save_Base_Reg = AM.Base_Reg;
1914 if (
auto *LoadN = dyn_cast<LoadSDNode>(Save_Base_Reg)) {
1916 if (matchLoadInAddress(LoadN, AM,
true))
1917 AM.Base_Reg = Save_Base_Reg;
1923 if (AM.Scale == 2 &&
1924 AM.BaseType == X86ISelAddressMode::RegBase &&
1925 AM.Base_Reg.getNode() ==
nullptr) {
1926 AM.Base_Reg = AM.IndexReg;
1933 (!AM.GV || !
TM.isLargeGlobalValue(AM.GV)) && Subtarget->is64Bit() &&
1934 AM.Scale == 1 && AM.BaseType == X86ISelAddressMode::RegBase &&
1935 AM.Base_Reg.getNode() ==
nullptr && AM.IndexReg.getNode() ==
nullptr &&
1937 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64);
1943bool X86DAGToDAGISel::matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
1949 X86ISelAddressMode Backup = AM;
1950 if (!matchAddressRecursively(
N.getOperand(0), AM,
Depth+1) &&
1951 !matchAddressRecursively(Handle.getValue().getOperand(1), AM,
Depth+1))
1956 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM,
1958 !matchAddressRecursively(Handle.getValue().getOperand(0), AM,
Depth + 1))
1965 if (AM.BaseType == X86ISelAddressMode::RegBase &&
1966 !AM.Base_Reg.getNode() &&
1967 !AM.IndexReg.getNode()) {
1968 N = Handle.getValue();
1969 AM.Base_Reg =
N.getOperand(0);
1970 AM.IndexReg =
N.getOperand(1);
1974 N = Handle.getValue();
1984 if (
N->getNodeId() == -1 ||
2004 X86ISelAddressMode &AM) {
2011 if (ScaleLog <= 0 || ScaleLog >= 4 ||
2012 Mask != (0xffu << ScaleLog))
2015 MVT XVT =
X.getSimpleValueType();
2016 MVT VT =
N.getSimpleValueType();
2041 AM.Scale = (1 << ScaleLog);
2049 X86ISelAddressMode &AM) {
2055 int64_t Mask = cast<ConstantSDNode>(
N->getOperand(1))->getSExtValue();
2060 bool FoundAnyExtend =
false;
2064 FoundAnyExtend =
true;
2082 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3)
2085 MVT VT =
N.getSimpleValueType();
2087 if (FoundAnyExtend) {
2108 AM.Scale = 1 << ShiftAmt;
2109 AM.IndexReg = NewAnd;
2143 X86ISelAddressMode &AM) {
2149 unsigned MaskIdx, MaskLen;
2152 unsigned MaskLZ = 64 - (MaskIdx + MaskLen);
2158 unsigned AMShiftAmt = MaskIdx;
2162 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2166 unsigned ScaleDown = (64 -
X.getSimpleValueType().getSizeInBits()) + ShiftAmt;
2167 if (MaskLZ < ScaleDown)
2169 MaskLZ -= ScaleDown;
2177 bool ReplacingAnyExtend =
false;
2179 unsigned ExtendBits =
X.getSimpleValueType().getSizeInBits() -
2180 X.getOperand(0).getSimpleValueType().getSizeInBits();
2183 X =
X.getOperand(0);
2184 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits;
2185 ReplacingAnyExtend =
true;
2187 APInt MaskedHighBits =
2194 MVT VT =
N.getSimpleValueType();
2195 if (ReplacingAnyExtend) {
2196 assert(
X.getValueType() != VT);
2203 MVT XVT =
X.getSimpleValueType();
2224 AM.Scale = 1 << AMShiftAmt;
2225 AM.IndexReg = NewExt;
2235 X86ISelAddressMode &AM,
2243 if (!Subtarget.hasTBM() &&
2244 !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR()))
2248 unsigned MaskIdx, MaskLen;
2256 unsigned AMShiftAmt = MaskIdx;
2260 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2262 MVT XVT =
X.getSimpleValueType();
2263 MVT VT =
N.getSimpleValueType();
2288 AM.Scale = 1 << AMShiftAmt;
2289 AM.IndexReg = NewExt;
2296 X86ISelAddressMode &AM,
2298 assert(AM.IndexReg.getNode() ==
nullptr &&
"IndexReg already matched");
2299 assert((AM.Scale == 1 || AM.Scale == 2 || AM.Scale == 4 || AM.Scale == 8) &&
2300 "Illegal index scale");
2306 EVT VT =
N.getValueType();
2307 unsigned Opc =
N.getOpcode();
2310 if (CurDAG->isBaseWithConstantOffset(
N)) {
2311 auto *AddVal = cast<ConstantSDNode>(
N.getOperand(1));
2313 if (!foldOffsetIntoAddress(
Offset, AM))
2314 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2318 if (Opc ==
ISD::ADD &&
N.getOperand(0) ==
N.getOperand(1)) {
2319 if (AM.Scale <= 4) {
2321 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2327 uint64_t ShiftAmt =
N.getConstantOperandVal(1);
2328 uint64_t ScaleAmt = 1ULL << ShiftAmt;
2329 if ((AM.Scale * ScaleAmt) <= 8) {
2330 AM.Scale *= ScaleAmt;
2331 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2339 if (Src.getOpcode() ==
ISD::ADD && Src->getFlags().hasNoSignedWrap() &&
2341 if (CurDAG->isBaseWithConstantOffset(Src)) {
2342 SDValue AddSrc = Src.getOperand(0);
2343 auto *AddVal = cast<ConstantSDNode>(Src.getOperand(1));
2345 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2347 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2353 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2354 CurDAG->RemoveDeadNode(
N.getNode());
2366 unsigned SrcOpc = Src.getOpcode();
2367 if (((SrcOpc ==
ISD::ADD && Src->getFlags().hasNoUnsignedWrap()) ||
2368 CurDAG->isADDLike(Src,
true)) &&
2370 if (CurDAG->isBaseWithConstantOffset(Src)) {
2371 SDValue AddSrc = Src.getOperand(0);
2373 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2384 if ((AM.Scale * ScaleAmt) <= 8 &&
2386 CurDAG->MaskedValueIsZero(ShVal, HiBits))) {
2387 AM.Scale *= ScaleAmt;
2388 SDValue ExtShVal = CurDAG->getNode(Opc,
DL, VT, ShVal);
2397 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2399 SDValue ExtAdd = CurDAG->getNode(SrcOpc,
DL, VT, ExtSrc, ExtVal);
2403 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2404 CurDAG->RemoveDeadNode(
N.getNode());
2405 return Res ? Res : ExtSrc;
2415bool X86DAGToDAGISel::matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
2419 dbgs() <<
"MatchAddress: ";
2424 return matchAddressBase(
N, AM);
2429 if (AM.isRIPRelative()) {
2433 if (!(AM.ES || AM.MCSym) && AM.JT != -1)
2436 if (
auto *Cst = dyn_cast<ConstantSDNode>(
N))
2437 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM))
2442 switch (
N.getOpcode()) {
2445 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
2446 if (
const auto *ESNode = dyn_cast<MCSymbolSDNode>(
N.getOperand(0))) {
2448 AM.MCSym = ESNode->getMCSymbol();
2454 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2455 if (!foldOffsetIntoAddress(Val, AM))
2462 if (!matchWrapper(
N, AM))
2467 if (!matchLoadInAddress(cast<LoadSDNode>(
N), AM))
2472 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2473 AM.Base_Reg.getNode() ==
nullptr &&
2475 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
2476 AM.Base_FrameIndex = cast<FrameIndexSDNode>(
N)->getIndex();
2482 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2485 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
2486 unsigned Val = CN->getZExtValue();
2491 if (Val == 1 || Val == 2 || Val == 3) {
2493 AM.Scale = 1 << Val;
2494 AM.IndexReg = matchIndexRecursively(ShVal, AM,
Depth + 1);
2502 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2506 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2507 "Unexpected value size!");
2516 if (!isa<ConstantSDNode>(
N.getOperand(1)) ||
2517 !isa<ConstantSDNode>(
And.getOperand(1)))
2519 uint64_t Mask =
And.getConstantOperandVal(1) >>
N.getConstantOperandVal(1);
2531 if (
N.getResNo() != 0)
break;
2536 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2537 AM.Base_Reg.getNode() ==
nullptr &&
2538 AM.IndexReg.getNode() ==
nullptr) {
2539 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
2540 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
2541 CN->getZExtValue() == 9) {
2542 AM.Scale =
unsigned(CN->getZExtValue())-1;
2553 auto *AddVal = cast<ConstantSDNode>(MulVal.
getOperand(1));
2554 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue();
2555 if (foldOffsetIntoAddress(Disp, AM))
2556 Reg =
N.getOperand(0);
2558 Reg =
N.getOperand(0);
2561 AM.IndexReg = AM.Base_Reg =
Reg;
2580 X86ISelAddressMode Backup = AM;
2581 if (matchAddressRecursively(
N.getOperand(0), AM,
Depth+1)) {
2582 N = Handle.getValue();
2586 N = Handle.getValue();
2588 if (AM.IndexReg.getNode() || AM.isRIPRelative()) {
2603 RHS.getOperand(0).getValueType() == MVT::i32))
2607 if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() &&
2608 !AM.Base_Reg.getNode()->hasOneUse()) ||
2609 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
2613 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
2614 ((AM.Disp != 0) && (Backup.Disp == 0)) +
2615 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
2627 AM.NegateIndex =
true;
2635 if (!CurDAG->isADDLike(
N))
2639 if (!matchAdd(
N, AM,
Depth))
2648 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2652 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2653 "Unexpected value size!");
2655 if (!isa<ConstantSDNode>(
N.getOperand(1)))
2658 if (
N.getOperand(0).getOpcode() ==
ISD::SRL) {
2687 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2697 AM.IndexReg =
Index;
2703 if (Src.getOpcode() ==
ISD::AND && Src.hasOneUse())
2704 if (
auto *MaskC = dyn_cast<ConstantSDNode>(Src.getOperand(1))) {
2705 Mask = MaskC->getAPIntValue();
2706 Src = Src.getOperand(0);
2709 if (Src.getOpcode() ==
ISD::SHL && Src.hasOneUse()) {
2711 SDValue ShlSrc = Src.getOperand(0);
2712 SDValue ShlAmt = Src.getOperand(1);
2713 auto *ShAmtC = dyn_cast<ConstantSDNode>(ShlAmt);
2716 unsigned ShAmtV = ShAmtC->getZExtValue();
2724 if (!Src->getFlags().hasNoUnsignedWrap() &&
2725 !CurDAG->MaskedValueIsZero(ShlSrc, HighZeros & Mask))
2733 MVT VT =
N.getSimpleValueType();
2737 if (!
Mask.isAllOnes()) {
2738 Res = CurDAG->getConstant(
Mask.lshr(ShAmtV),
DL, SrcVT);
2740 Res = CurDAG->getNode(
ISD::AND,
DL, SrcVT, ShlSrc, Res);
2747 CurDAG->ReplaceAllUsesWith(
N, NewShl);
2748 CurDAG->RemoveDeadNode(
N.getNode());
2751 AM.Scale = 1 << ShAmtV;
2755 AM.IndexReg = matchIndexRecursively(Zext, AM,
Depth + 1);
2759 if (Src.getOpcode() ==
ISD::SRL && !
Mask.isAllOnes()) {
2762 Src.getOperand(0), AM))
2767 Src.getOperand(0), AM))
2772 Src.getOperand(0), AM, *Subtarget))
2780 return matchAddressBase(
N, AM);
2785bool X86DAGToDAGISel::matchAddressBase(
SDValue N, X86ISelAddressMode &AM) {
2787 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
2789 if (!AM.IndexReg.getNode()) {
2800 AM.BaseType = X86ISelAddressMode::RegBase;
2805bool X86DAGToDAGISel::matchVectorAddressRecursively(
SDValue N,
2806 X86ISelAddressMode &AM,
2810 dbgs() <<
"MatchVectorAddress: ";
2815 return matchAddressBase(
N, AM);
2818 switch (
N.getOpcode()) {
2820 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2821 if (!foldOffsetIntoAddress(Val, AM))
2826 if (!matchWrapper(
N, AM))
2834 X86ISelAddressMode Backup = AM;
2835 if (!matchVectorAddressRecursively(
N.getOperand(0), AM,
Depth + 1) &&
2836 !matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2842 if (!matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2844 !matchVectorAddressRecursively(Handle.getValue().getOperand(0), AM,
2849 N = Handle.getValue();
2854 return matchAddressBase(
N, AM);
2860bool X86DAGToDAGISel::matchVectorAddress(
SDValue N, X86ISelAddressMode &AM) {
2861 return matchVectorAddressRecursively(
N, AM, 0);
2869 X86ISelAddressMode AM;
2875 AM.IndexReg = matchIndexRecursively(IndexOp, AM, 0);
2877 AM.IndexReg = IndexOp;
2881 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2883 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2885 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2891 if (matchVectorAddress(BasePtr, AM))
2894 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2908 X86ISelAddressMode AM;
2920 unsigned AddrSpace =
2921 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
2923 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2925 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2927 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2932 MVT VT =
N.getSimpleValueType();
2934 if (matchAddress(
N, AM))
2937 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2953 N =
N.getOperand(0);
2968 const GlobalValue *GV = cast<GlobalAddressSDNode>(
N)->getGlobal();
2970 return CR->getUnsignedMax().ult(1ull << 32);
2972 return !
TM.isLargeGlobalValue(GV);
2981 if (!selectLEAAddr(
N,
Base, Scale,
Index, Disp, Segment))
2984 auto *
RN = dyn_cast<RegisterSDNode>(
Base);
2985 if (RN &&
RN->getReg() == 0)
2986 Base = CurDAG->getRegister(0, MVT::i64);
2987 else if (
Base.getValueType() == MVT::i32 && !isa<FrameIndexSDNode>(
Base)) {
2991 Base = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
2995 RN = dyn_cast<RegisterSDNode>(
Index);
2996 if (RN &&
RN->getReg() == 0)
2997 Index = CurDAG->getRegister(0, MVT::i64);
3000 "Expect to be extending 32-bit registers for use in LEA");
3003 Index = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
3012bool X86DAGToDAGISel::selectLEAAddr(
SDValue N,
3016 X86ISelAddressMode AM;
3020 MVT VT =
N.getSimpleValueType();
3025 SDValue T = CurDAG->getRegister(0, MVT::i32);
3027 if (matchAddress(
N, AM))
3032 unsigned Complexity = 0;
3033 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode())
3035 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
3038 if (AM.IndexReg.getNode())
3051 if (AM.hasSymbolicDisplacement()) {
3053 if (Subtarget->is64Bit())
3063 auto isMathWithFlags = [](
SDValue V) {
3064 switch (
V.getOpcode()) {
3085 if (isMathWithFlags(
N.getOperand(0)) || isMathWithFlags(
N.getOperand(1)))
3093 if (Complexity <= 2)
3096 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
3107 X86ISelAddressMode AM;
3108 if (
auto *GA = dyn_cast<GlobalAddressSDNode>(
N)) {
3109 AM.GV = GA->getGlobal();
3110 AM.Disp += GA->getOffset();
3111 AM.SymbolFlags = GA->getTargetFlags();
3113 auto *SA = cast<ExternalSymbolSDNode>(
N);
3114 AM.ES = SA->getSymbol();
3115 AM.SymbolFlags = SA->getTargetFlags();
3118 if (Subtarget->is32Bit()) {
3120 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
3123 MVT VT =
N.getSimpleValueType();
3124 getAddressOperands(AM,
SDLoc(
N), VT,
Base, Scale,
Index, Disp, Segment);
3132 EVT VT =
N.getValueType();
3133 bool WasTruncated =
false;
3135 WasTruncated =
true;
3136 N =
N.getOperand(0);
3145 unsigned Opc =
N.getOperand(0)->getOpcode();
3147 Op =
N.getOperand(0);
3150 return !WasTruncated;
3154 auto *GA = cast<GlobalAddressSDNode>(
N.getOperand(0));
3155 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange();
3156 if (!CR || CR->getUnsignedMax().uge(1ull << VT.
getSizeInBits()))
3160 Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(),
SDLoc(
N), VT,
3161 GA->getOffset(), GA->getTargetFlags());
3169 assert(Root &&
P &&
"Unknown root/parent nodes");
3171 !IsProfitableToFold(
N,
P, Root) ||
3172 !IsLegalToFold(
N,
P, Root, OptLevel))
3175 return selectAddr(
N.getNode(),
3176 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3183 assert(Root &&
P &&
"Unknown root/parent nodes");
3185 !IsProfitableToFold(
N,
P, Root) ||
3186 !IsLegalToFold(
N,
P, Root, OptLevel))
3189 return selectAddr(
N.getNode(),
3190 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3196SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
3197 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
3199 return CurDAG->getRegister(GlobalBaseReg, TLI->
getPointerTy(
DL)).getNode();
3202bool X86DAGToDAGISel::isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const {
3204 N =
N->getOperand(0).getNode();
3208 auto *GA = dyn_cast<GlobalAddressSDNode>(
N->getOperand(0));
3212 auto *GV = GA->getGlobal();
3215 return CR->getSignedMin().sge(-1ull << Width) &&
3216 CR->getSignedMax().slt(1ull << Width);
3221 return Width == 32 && !
TM.isLargeGlobalValue(GV);
3225 assert(
N->isMachineOpcode() &&
"Unexpected node");
3226 unsigned Opc =
N->getMachineOpcode();
3227 const MCInstrDesc &MCID = getInstrInfo()->get(Opc);
3232 return static_cast<X86::CondCode>(
N->getConstantOperandVal(CondNo));
3237bool X86DAGToDAGISel::onlyUsesZeroFlag(
SDValue Flags)
const {
3242 if (UI.getUse().getResNo() !=
Flags.getResNo())
3246 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3250 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3252 if (FlagUI.getUse().getResNo() != 1)
continue;
3254 if (!FlagUI->isMachineOpcode())
return false;
3273bool X86DAGToDAGISel::hasNoSignFlagUses(
SDValue Flags)
const {
3278 if (UI.getUse().getResNo() !=
Flags.getResNo())
3282 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3286 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3288 if (FlagUI.getUse().getResNo() != 1)
continue;
3290 if (!FlagUI->isMachineOpcode())
return false;
3329 bool X86DAGToDAGISel::hasNoCarryFlagUses(
SDValue Flags)
const {
3334 if (UI.getUse().getResNo() !=
Flags.getResNo())
3337 unsigned UIOpc = UI->getOpcode();
3341 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3345 FlagUI != FlagUE; ++FlagUI) {
3347 if (FlagUI.getUse().getResNo() != 1)
3350 if (!FlagUI->isMachineOpcode())
3391 if (StoredVal.
getResNo() != 0)
return false;
3405 LoadNode = cast<LoadSDNode>(Load);
3408 if (!Load.hasOneUse())
3416 bool FoundLoad =
false;
3420 const unsigned int Max = 1024;
3462 if (Chain == Load.getValue(1)) {
3468 if (
Op == Load.getValue(1)) {
3484 if (
Op.getNode() != LoadNode)
3517bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(
SDNode *
Node) {
3518 auto *StoreNode = cast<StoreSDNode>(
Node);
3525 EVT MemVT = StoreNode->getMemoryVT();
3526 if (MemVT != MVT::i64 && MemVT != MVT::i32 && MemVT != MVT::i16 &&
3530 bool IsCommutable =
false;
3531 bool IsNegate =
false;
3545 IsCommutable =
true;
3549 unsigned LoadOpNo = IsNegate ? 1 : 0;
3553 LoadNode, InputChain)) {
3560 LoadNode, InputChain))
3569 auto SelectOpcode = [&](
unsigned Opc64,
unsigned Opc32,
unsigned Opc16,
3590 unsigned NewOpc = SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m,
3600 if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) {
3604 if ((IsOne || IsNegOne) && hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3607 ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
3608 : SelectOpcode(
X86::DEC64m,
X86::DEC32m,
X86::DEC16m,
X86::DEC8m);
3621 auto SelectRegOpcode = [SelectOpcode](
unsigned Opc) {
3624 return SelectOpcode(X86::ADD64mr, X86::ADD32mr, X86::ADD16mr,
3627 return SelectOpcode(X86::ADC64mr, X86::ADC32mr, X86::ADC16mr,
3630 return SelectOpcode(X86::SUB64mr, X86::SUB32mr, X86::SUB16mr,
3633 return SelectOpcode(X86::SBB64mr, X86::SBB32mr, X86::SBB16mr,
3636 return SelectOpcode(X86::AND64mr, X86::AND32mr, X86::AND16mr,
3639 return SelectOpcode(X86::OR64mr, X86::OR32mr, X86::OR16mr, X86::OR8mr);
3641 return SelectOpcode(X86::XOR64mr, X86::XOR32mr, X86::XOR16mr,
3647 auto SelectImmOpcode = [SelectOpcode](
unsigned Opc) {
3650 return SelectOpcode(X86::ADD64mi32, X86::ADD32mi, X86::ADD16mi,
3653 return SelectOpcode(X86::ADC64mi32, X86::ADC32mi, X86::ADC16mi,
3656 return SelectOpcode(X86::SUB64mi32, X86::SUB32mi, X86::SUB16mi,
3659 return SelectOpcode(X86::SBB64mi32, X86::SBB32mi, X86::SBB16mi,
3662 return SelectOpcode(X86::AND64mi32, X86::AND32mi, X86::AND16mi,
3665 return SelectOpcode(X86::OR64mi32, X86::OR32mi, X86::OR16mi,
3668 return SelectOpcode(X86::XOR64mi32, X86::XOR32mi, X86::XOR16mi,
3675 unsigned NewOpc = SelectRegOpcode(Opc);
3680 if (
auto *OperandC = dyn_cast<ConstantSDNode>(Operand)) {
3681 int64_t OperandV = OperandC->getSExtValue();
3687 ((MemVT != MVT::i8 && !isInt<8>(OperandV) && isInt<8>(-OperandV)) ||
3688 (MemVT == MVT::i64 && !isInt<32>(OperandV) &&
3689 isInt<32>(-OperandV))) &&
3690 hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3691 OperandV = -OperandV;
3695 if (MemVT != MVT::i64 || isInt<32>(OperandV)) {
3696 Operand = CurDAG->getTargetConstant(OperandV,
SDLoc(
Node), MemVT);
3697 NewOpc = SelectImmOpcode(Opc);
3703 CurDAG->getCopyToReg(InputChain,
SDLoc(
Node), X86::EFLAGS,
3707 Segment, Operand, CopyTo, CopyTo.
getValue(1)};
3708 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3712 Segment, Operand, InputChain};
3713 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3724 CurDAG->setNodeMemRefs(Result, MemOps);
3730 CurDAG->RemoveDeadNode(
Node);
3741bool X86DAGToDAGISel::matchBitExtract(
SDNode *
Node) {
3745 "Should be either an and-mask, or right-shift after clearing high bits.");
3748 if (!Subtarget->hasBMI() && !Subtarget->hasBMI2())
3751 MVT NVT =
Node->getSimpleValueType(0);
3754 if (NVT != MVT::i32 && NVT != MVT::i64)
3762 const bool AllowExtraUsesByDefault = Subtarget->hasBMI2();
3763 auto checkUses = [AllowExtraUsesByDefault](
3765 std::optional<bool> AllowExtraUses) {
3766 return AllowExtraUses.value_or(AllowExtraUsesByDefault) ||
3767 Op.getNode()->hasNUsesOfValue(NUses,
Op.getResNo());
3769 auto checkOneUse = [checkUses](
SDValue Op,
3770 std::optional<bool> AllowExtraUses =
3772 return checkUses(
Op, 1, AllowExtraUses);
3774 auto checkTwoUse = [checkUses](
SDValue Op,
3775 std::optional<bool> AllowExtraUses =
3777 return checkUses(
Op, 2, AllowExtraUses);
3780 auto peekThroughOneUseTruncation = [checkOneUse](
SDValue V) {
3782 assert(
V.getSimpleValueType() == MVT::i32 &&
3783 V.getOperand(0).getSimpleValueType() == MVT::i64 &&
3784 "Expected i64 -> i32 truncation");
3785 V =
V.getOperand(0);
3791 auto matchPatternA = [checkOneUse, peekThroughOneUseTruncation, &NBits,
3794 if (
Mask->getOpcode() !=
ISD::ADD || !checkOneUse(Mask))
3800 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3805 NBits =
M0->getOperand(1);
3806 NegateNBits =
false;
3810 auto isAllOnes = [
this, peekThroughOneUseTruncation, NVT](
SDValue V) {
3811 V = peekThroughOneUseTruncation(V);
3812 return CurDAG->MaskedValueIsAllOnes(
3818 auto matchPatternB = [checkOneUse, isAllOnes, peekThroughOneUseTruncation,
3821 if (
Mask.getOpcode() !=
ISD::XOR || !checkOneUse(Mask))
3824 if (!isAllOnes(
Mask->getOperand(1)))
3827 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3831 if (!isAllOnes(
M0->getOperand(0)))
3833 NBits =
M0->getOperand(1);
3834 NegateNBits =
false;
3840 auto canonicalizeShiftAmt = [&NBits, &NegateNBits](
SDValue ShiftAmt,
3841 unsigned Bitwidth) {
3846 NBits = NBits.getOperand(0);
3851 auto *V0 = dyn_cast<ConstantSDNode>(NBits.getOperand(0));
3852 if (!V0 || V0->getZExtValue() != Bitwidth)
3854 NBits = NBits.getOperand(1);
3855 NegateNBits =
false;
3861 auto matchPatternC = [checkOneUse, peekThroughOneUseTruncation, &NegateNBits,
3864 Mask = peekThroughOneUseTruncation(Mask);
3865 unsigned Bitwidth =
Mask.getSimpleValueType().getSizeInBits();
3867 if (
Mask.getOpcode() !=
ISD::SRL || !checkOneUse(Mask))
3874 if (!checkOneUse(
M1))
3876 canonicalizeShiftAmt(
M1, Bitwidth);
3881 return !NegateNBits;
3889 auto matchPatternD = [checkOneUse, checkTwoUse, canonicalizeShiftAmt,
3890 AllowExtraUsesByDefault, &NegateNBits,
3903 canonicalizeShiftAmt(N1, Bitwidth);
3907 const bool AllowExtraUses = AllowExtraUsesByDefault && !NegateNBits;
3908 if (!checkOneUse(N0, AllowExtraUses) || !checkTwoUse(N1, AllowExtraUses))
3914 auto matchLowBitMask = [matchPatternA, matchPatternB,
3916 return matchPatternA(Mask) || matchPatternB(Mask) || matchPatternC(Mask);
3920 X =
Node->getOperand(0);
3923 if (matchLowBitMask(Mask)) {
3927 if (!matchLowBitMask(Mask))
3931 X = CurDAG->getAllOnesConstant(
SDLoc(
Node), NVT);
3932 }
else if (!matchPatternD(
Node))
3937 if (NegateNBits && !Subtarget->hasBMI2())
3949 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, MVT::i32), 0);
3952 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit,
DL, MVT::i32);
3954 NBits =
SDValue(CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG,
DL,
3955 MVT::i32, ImplDef, NBits, SRIdxVal),
3965 NBits = CurDAG->getNode(
ISD::SUB,
DL, MVT::i32, BitWidthC, NBits);
3969 if (Subtarget->hasBMI2()) {
3971 if (NVT != MVT::i32) {
3979 SelectCode(Extract.
getNode());
3988 SDValue RealX = peekThroughOneUseTruncation(
X);
3994 MVT XVT =
X.getSimpleValueType();
4004 SDValue C8 = CurDAG->getConstant(8,
DL, MVT::i8);
4012 SDValue ShiftAmt =
X.getOperand(1);
4013 X =
X.getOperand(0);
4016 "Expected shift amount to be i8");
4020 SDValue OrigShiftAmt = ShiftAmt;
4025 Control = CurDAG->getNode(
ISD::OR,
DL, MVT::i32, Control, ShiftAmt);
4030 if (XVT != MVT::i32) {
4045 SelectCode(Extract.
getNode());
4052 MVT NVT =
Node->getSimpleValueType(0);
4065 Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR());
4066 if (!PreferBEXTR && !Subtarget->hasBMI2())
4078 if (NVT != MVT::i32 && NVT != MVT::i64)
4082 auto *MaskCst = dyn_cast<ConstantSDNode>(N1);
4083 auto *ShiftCst = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
4084 if (!MaskCst || !ShiftCst)
4092 uint64_t Shift = ShiftCst->getZExtValue();
4097 if (Shift == 8 && MaskSize == 8)
4108 if (!PreferBEXTR && MaskSize <= 32)
4112 unsigned ROpc, MOpc;
4114#define GET_EGPR_IF_ENABLED(OPC) (Subtarget->hasEGPR() ? OPC##_EVEX : OPC)
4116 assert(Subtarget->hasBMI2() &&
"We must have BMI2's BZHI then.");
4120 Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT);
4125 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4126 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4132 Control = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT);
4133 if (Subtarget->hasTBM()) {
4134 ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri;
4135 MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi;
4137 assert(Subtarget->hasBMI() &&
"We must have BMI1's BEXTR then.");
4143 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4144 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4150 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4151 if (tryFoldLoad(
Node, N0.
getNode(), Input, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4153 Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Control, Input.
getOperand(0)};
4154 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
4155 NewNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4159 CurDAG->setNodeMemRefs(NewNode, {cast<LoadSDNode>(Input)->getMemOperand()});
4161 NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, Control);
4166 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT);
4170 CurDAG->getMachineNode(NewOpc, dl, NVT,
SDValue(NewNode, 0), ShAmt);
4177MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
4178 bool MayFoldLoad,
const SDLoc &dl,
4183 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4184 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4187 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4188 if (MayFoldLoad && tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4189 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4191 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other);
4192 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4196 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
4201 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32);
4202 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4209MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
4210 bool MayFoldLoad,
const SDLoc &dl,
4216 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4217 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4220 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4221 if (MayFoldLoad && tryFoldLoad(
Node, N2, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4222 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4224 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other, MVT::Glue);
4225 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4230 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N2)->getMemOperand()});
4235 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Glue);
4236 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4241bool X86DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
4242 EVT VT =
N->getValueType(0);
4249 unsigned Size = VT == MVT::i64 ? 64 : 32;
4251 SDValue OrigShiftAmt =
N->getOperand(1);
4252 SDValue ShiftAmt = OrigShiftAmt;
4267 auto *Add0C = dyn_cast<ConstantSDNode>(Add0);
4268 auto *Add1C = dyn_cast<ConstantSDNode>(Add1);
4271 if (Add1C && Add1C->getAPIntValue().urem(
Size) == 0) {
4275 ((Add0C && Add0C->getAPIntValue().urem(
Size) ==
Size - 1) ||
4276 (Add1C && Add1C->getAPIntValue().urem(
Size) ==
Size - 1))) {
4280 assert(Add0C ==
nullptr || Add1C ==
nullptr);
4289 NewShiftAmt = CurDAG->getNode(
ISD::XOR,
DL, OpVT,
4290 Add0C ==
nullptr ? Add0 : Add1,
AllOnes);
4296 Add0C->getZExtValue() != 0) {
4299 if (Add0C->getZExtValue() %
Size == 0)
4302 Add0C->getZExtValue() % 32 == 0) {
4310 Add0 = CurDAG->getZExtOrTrunc(Add0,
DL, SubVT);
4314 X = CurDAG->getNode(
ISD::ADD,
DL, SubVT, Add1, Add0);
4336 NewShiftAmt = CurDAG->getNode(
ISD::TRUNCATE,
DL, MVT::i8, NewShiftAmt);
4343 NewShiftAmt = CurDAG->getNode(
ISD::AND,
DL, MVT::i8, NewShiftAmt,
4344 CurDAG->getConstant(
Size - 1,
DL, MVT::i8));
4348 SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(
N,
N->getOperand(0),
4350 if (UpdatedNode !=
N) {
4353 ReplaceNode(
N, UpdatedNode);
4360 CurDAG->RemoveDeadNode(OrigShiftAmt.
getNode());
4368bool X86DAGToDAGISel::tryShrinkShlLogicImm(
SDNode *
N) {
4369 MVT NVT =
N->getSimpleValueType(0);
4370 unsigned Opcode =
N->getOpcode();
4378 auto *Cst = dyn_cast<ConstantSDNode>(N1);
4382 int64_t Val = Cst->getSExtValue();
4387 bool FoundAnyExtend =
false;
4391 FoundAnyExtend =
true;
4399 if (NVT != MVT::i32 && NVT != MVT::i64)
4402 auto *ShlCst = dyn_cast<ConstantSDNode>(Shift.
getOperand(1));
4406 uint64_t ShAmt = ShlCst->getZExtValue();
4410 uint64_t RemovedBitsMask = (1ULL << ShAmt) - 1;
4411 if (Opcode !=
ISD::AND && (Val & RemovedBitsMask) != 0)
4416 auto CanShrinkImmediate = [&](int64_t &ShiftedVal) {
4420 ShiftedVal = (
uint64_t)Val >> ShAmt;
4421 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4424 if (ShiftedVal == UINT8_MAX || ShiftedVal == UINT16_MAX)
4427 ShiftedVal = Val >> ShAmt;
4428 if ((!isInt<8>(Val) && isInt<8>(ShiftedVal)) ||
4429 (!isInt<32>(Val) && isInt<32>(ShiftedVal)))
4433 ShiftedVal = (
uint64_t)Val >> ShAmt;
4434 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4441 if (!CanShrinkImmediate(ShiftedVal))
4451 unsigned ZExtWidth = Cst->getAPIntValue().getActiveBits();
4457 NeededMask &= ~Cst->getAPIntValue();
4459 if (CurDAG->MaskedValueIsZero(
N->getOperand(0), NeededMask))
4464 if (FoundAnyExtend) {
4470 SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT);
4472 SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT,
X, NewCst);
4481bool X86DAGToDAGISel::matchVPTERNLOG(
SDNode *Root,
SDNode *ParentA,
4485 assert(
A.isOperandOf(ParentA) &&
B.isOperandOf(ParentB) &&
4486 C.isOperandOf(ParentC) &&
"Incorrect parent node");
4488 auto tryFoldLoadOrBCast =
4491 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4497 L =
L.getOperand(0);
4504 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4505 unsigned Size = MemIntr->getMemoryVT().getSizeInBits();
4509 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4512 bool FoldedLoad =
false;
4513 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4514 if (tryFoldLoadOrBCast(Root, ParentC,
C, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4516 }
else if (tryFoldLoadOrBCast(Root, ParentA,
A, Tmp0, Tmp1, Tmp2, Tmp3,
4521 uint8_t OldImm =
Imm;
4522 Imm = OldImm & 0xa5;
4523 if (OldImm & 0x02)
Imm |= 0x10;
4524 if (OldImm & 0x10)
Imm |= 0x02;
4525 if (OldImm & 0x08)
Imm |= 0x40;
4526 if (OldImm & 0x40)
Imm |= 0x08;
4527 }
else if (tryFoldLoadOrBCast(Root, ParentB,
B, Tmp0, Tmp1, Tmp2, Tmp3,
4532 uint8_t OldImm =
Imm;
4533 Imm = OldImm & 0x99;
4534 if (OldImm & 0x02)
Imm |= 0x04;
4535 if (OldImm & 0x04)
Imm |= 0x02;
4536 if (OldImm & 0x20)
Imm |= 0x40;
4537 if (OldImm & 0x40)
Imm |= 0x20;
4542 SDValue TImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
4548 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
4552 auto *MemIntr = cast<MemIntrinsicSDNode>(
C);
4553 unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits();
4554 assert((EltSize == 32 || EltSize == 64) &&
"Unexpected broadcast size!");
4556 bool UseD = EltSize == 32;
4558 Opc = UseD ? X86::VPTERNLOGDZ128rmbi : X86::VPTERNLOGQZ128rmbi;
4560 Opc = UseD ? X86::VPTERNLOGDZ256rmbi : X86::VPTERNLOGQZ256rmbi;
4562 Opc = UseD ? X86::VPTERNLOGDZrmbi : X86::VPTERNLOGQZrmbi;
4568 Opc = UseD ? X86::VPTERNLOGDZ128rmi : X86::VPTERNLOGQZ128rmi;
4570 Opc = UseD ? X86::VPTERNLOGDZ256rmi : X86::VPTERNLOGQZ256rmi;
4572 Opc = UseD ? X86::VPTERNLOGDZrmi : X86::VPTERNLOGQZrmi;
4577 SDValue Ops[] = {
A,
B, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, TImm,
C.getOperand(0)};
4578 MNode = CurDAG->getMachineNode(Opc,
DL, VTs, Ops);
4581 ReplaceUses(
C.getValue(1),
SDValue(MNode, 1));
4583 CurDAG->setNodeMemRefs(MNode, {cast<MemSDNode>(
C)->getMemOperand()});
4588 Opc = UseD ? X86::VPTERNLOGDZ128rri : X86::VPTERNLOGQZ128rri;
4590 Opc = UseD ? X86::VPTERNLOGDZ256rri : X86::VPTERNLOGQZ256rri;
4592 Opc = UseD ? X86::VPTERNLOGDZrri : X86::VPTERNLOGQZrri;
4596 MNode = CurDAG->getMachineNode(Opc,
DL, NVT, {
A,
B,
C, TImm});
4600 CurDAG->RemoveDeadNode(Root);
4606bool X86DAGToDAGISel::tryVPTERNLOG(
SDNode *
N) {
4607 MVT NVT =
N->getSimpleValueType(0);
4610 if (!NVT.
isVector() || !Subtarget->hasAVX512() ||
4621 auto getFoldableLogicOp = [](
SDValue Op) {
4624 Op =
Op.getOperand(0);
4626 if (!
Op.hasOneUse())
4629 unsigned Opc =
Op.getOpcode();
4638 if ((FoldableOp = getFoldableLogicOp(N1))) {
4640 }
else if ((FoldableOp = getFoldableLogicOp(N0))) {
4653 uint8_t TernlogMagicA = 0xf0;
4654 uint8_t TernlogMagicB = 0xcc;
4655 uint8_t TernlogMagicC = 0xaa;
4664 Parent =
Op.getNode();
4665 Op =
Op.getOperand(0);
4669 PeekThroughNot(
A, ParentA, TernlogMagicA);
4670 PeekThroughNot(
B, ParentB, TernlogMagicB);
4671 PeekThroughNot(
C, ParentC, TernlogMagicC);
4676 case ISD::AND:
Imm = TernlogMagicB & TernlogMagicC;
break;
4677 case ISD::OR:
Imm = TernlogMagicB | TernlogMagicC;
break;
4678 case ISD::XOR:
Imm = TernlogMagicB ^ TernlogMagicC;
break;
4682 switch (
N->getOpcode()) {
4686 Imm &= ~TernlogMagicA;
4688 Imm = ~(
Imm) & TernlogMagicA;
4695 return matchVPTERNLOG(
N, ParentA, ParentB, ParentC,
A,
B,
C, Imm);
4705bool X86DAGToDAGISel::shrinkAndImmediate(
SDNode *
And) {
4708 MVT VT =
And->getSimpleValueType(0);
4709 if (VT != MVT::i32 && VT != MVT::i64)
4712 auto *And1C = dyn_cast<ConstantSDNode>(
And->getOperand(1));
4721 APInt MaskVal = And1C->getAPIntValue();
4723 if (!MaskLZ || (VT == MVT::i64 && MaskLZ == 32))
4727 if (VT == MVT::i64 && MaskLZ >= 32) {
4729 MaskVal = MaskVal.
trunc(32);
4734 APInt NegMaskVal = MaskVal | HighZeros;
4743 if (VT == MVT::i64 && MaskVal.
getBitWidth() < 64) {
4744 NegMaskVal = NegMaskVal.
zext(64);
4745 HighZeros = HighZeros.
zext(64);
4750 if (!CurDAG->MaskedValueIsZero(And0, HighZeros))
4770 bool FoldedBCast,
bool Masked) {
4771#define VPTESTM_CASE(VT, SUFFIX) \
4774 return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
4775 return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
4778#define VPTESTM_BROADCAST_CASES(SUFFIX) \
4779default: llvm_unreachable("Unexpected VT!"); \
4780VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
4781VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
4782VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
4783VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
4784VPTESTM_CASE(v16i32, DZ##SUFFIX) \
4785VPTESTM_CASE(v8i64, QZ##SUFFIX)
4787#define VPTESTM_FULL_CASES(SUFFIX) \
4788VPTESTM_BROADCAST_CASES(SUFFIX) \
4789VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
4790VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
4791VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
4792VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
4793VPTESTM_CASE(v64i8, BZ##SUFFIX) \
4794VPTESTM_CASE(v32i16, WZ##SUFFIX)
4812#undef VPTESTM_FULL_CASES
4813#undef VPTESTM_BROADCAST_CASES
4819bool X86DAGToDAGISel::tryVPTESTM(
SDNode *Root,
SDValue Setcc,
4821 assert(Subtarget->hasAVX512() &&
"Expected AVX512!");
4871 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4876 if (CmpSVT != MVT::i32 && CmpSVT != MVT::i64)
4882 L =
L.getOperand(0);
4888 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4889 if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.
getSizeInBits())
4892 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4896 bool CanFoldLoads = Src0 != Src1;
4898 bool FoldedLoad =
false;
4899 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4901 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src1, Tmp0, Tmp1, Tmp2,
4905 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src0, Tmp0, Tmp1,
4914 bool IsMasked = InMask.
getNode() !=
nullptr;
4927 SDValue ImplDef =
SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl,
4929 Src0 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src0);
4932 Src1 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src1);
4937 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4938 InMask =
SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4939 dl, MaskVT, InMask, RC), 0);
4944 unsigned Opc =
getVPTESTMOpc(CmpVT, IsTestN, FoldedLoad, FoldedBCast,
4949 SDVTList VTs = CurDAG->getVTList(MaskVT, MVT::Other);
4952 SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4954 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
4956 SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4958 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
4964 CurDAG->setNodeMemRefs(CNode, {cast<MemSDNode>(Src1)->getMemOperand()});
4967 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, InMask, Src0, Src1);
4969 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, Src0, Src1);
4975 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4976 CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4977 dl, ResVT,
SDValue(CNode, 0), RC);
4981 CurDAG->RemoveDeadNode(Root);
4987bool X86DAGToDAGISel::tryMatchBitSelect(
SDNode *
N) {
4990 MVT NVT =
N->getSimpleValueType(0);
4993 if (!NVT.
isVector() || !Subtarget->hasAVX512())
5027 SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8);
5036 MVT NVT =
Node->getSimpleValueType(0);
5037 unsigned Opcode =
Node->getOpcode();
5040 if (
Node->isMachineOpcode()) {
5042 Node->setNodeId(-1);
5049 unsigned IntNo =
Node->getConstantOperandVal(1);
5052 case Intrinsic::x86_encodekey128:
5053 case Intrinsic::x86_encodekey256: {
5054 if (!Subtarget->hasKL())
5060 case Intrinsic::x86_encodekey128:
5061 Opcode = X86::ENCODEKEY128;
5063 case Intrinsic::x86_encodekey256:
5064 Opcode = X86::ENCODEKEY256;
5069 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(3),
5071 if (Opcode == X86::ENCODEKEY256)
5072 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(4),
5076 Opcode, dl,
Node->getVTList(),
5077 {Node->getOperand(2), Chain, Chain.getValue(1)});
5078 ReplaceNode(
Node, Res);
5081 case Intrinsic::x86_tileloadd64_internal:
5082 case Intrinsic::x86_tileloaddt164_internal: {
5083 if (!Subtarget->hasAMXTILE())
5085 unsigned Opc = IntNo == Intrinsic::x86_tileloadd64_internal
5087 : X86::PTILELOADDT1V;
5090 SDValue Scale = getI8Imm(1, dl);
5092 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5093 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5097 Node->getOperand(3),
5104 CNode = CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops);
5105 ReplaceNode(
Node, CNode);
5112 unsigned IntNo =
Node->getConstantOperandVal(1);
5115 case Intrinsic::x86_sse3_monitor:
5116 case Intrinsic::x86_monitorx:
5117 case Intrinsic::x86_clzero: {
5118 bool Use64BitPtr =
Node->getOperand(2).getValueType() == MVT::i64;
5123 case Intrinsic::x86_sse3_monitor:
5124 if (!Subtarget->hasSSE3())
5126 Opc = Use64BitPtr ? X86::MONITOR64rrr : X86::MONITOR32rrr;
5128 case Intrinsic::x86_monitorx:
5129 if (!Subtarget->hasMWAITX())
5131 Opc = Use64BitPtr ? X86::MONITORX64rrr : X86::MONITORX32rrr;
5133 case Intrinsic::x86_clzero:
5134 if (!Subtarget->hasCLZERO())
5136 Opc = Use64BitPtr ? X86::CLZERO64r : X86::CLZERO32r;
5141 unsigned PtrReg = Use64BitPtr ? X86::RAX : X86::EAX;
5142 SDValue Chain = CurDAG->getCopyToReg(
Node->getOperand(0), dl, PtrReg,
5146 if (IntNo == Intrinsic::x86_sse3_monitor ||
5147 IntNo == Intrinsic::x86_monitorx) {
5149 Chain = CurDAG->getCopyToReg(Chain, dl, X86::ECX,
Node->getOperand(3),
5152 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EDX,
Node->getOperand(4),
5157 MachineSDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
5159 ReplaceNode(
Node, CNode);
5165 case Intrinsic::x86_tilestored64_internal: {
5166 unsigned Opc = X86::PTILESTOREDV;
5169 SDValue Scale = getI8Imm(1, dl);
5171 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5172 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5176 Node->getOperand(3),
5182 Node->getOperand(6),
5184 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5185 ReplaceNode(
Node, CNode);
5188 case Intrinsic::x86_tileloadd64:
5189 case Intrinsic::x86_tileloaddt164:
5190 case Intrinsic::x86_tilestored64: {
5191 if (!Subtarget->hasAMXTILE())
5196 case Intrinsic::x86_tileloadd64: Opc = X86::PTILELOADD;
break;
5197 case Intrinsic::x86_tileloaddt164: Opc = X86::PTILELOADDT1;
break;
5198 case Intrinsic::x86_tilestored64: Opc = X86::PTILESTORED;
break;
5201 unsigned TIndex =
Node->getConstantOperandVal(2);
5202 SDValue TReg = getI8Imm(TIndex, dl);
5204 SDValue Scale = getI8Imm(1, dl);
5206 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5207 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5210 if (Opc == X86::PTILESTORED) {
5212 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5215 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5217 ReplaceNode(
Node, CNode);
5225 if (Subtarget->isTargetNaCl())
5229 if (Subtarget->isTarget64BitILP32()) {
5234 assert(
Target.getValueType() == MVT::i32 &&
"Unexpected VT!");
5235 SDValue ZextTarget = CurDAG->getZExtOrTrunc(
Target, dl, MVT::i64);
5236 SDValue Brind = CurDAG->getNode(Opcode, dl, MVT::Other,
5237 Node->getOperand(0), ZextTarget);
5239 SelectCode(ZextTarget.
getNode());
5246 ReplaceNode(
Node, getGlobalBaseReg());
5254 CurDAG->RemoveDeadNode(
Node);
5260 if (matchBitExtract(
Node))
5265 if (tryShiftAmountMod(
Node))
5270 uint8_t
Imm =
Node->getConstantOperandVal(3);
5272 Node->getOperand(1),
Node->getOperand(2), Imm))
5278 if (tryVPTERNLOG(
Node))
5288 tryVPTESTM(
Node, N0, N1))
5291 tryVPTESTM(
Node, N1, N0))
5297 CurDAG->RemoveDeadNode(
Node);
5300 if (matchBitExtract(
Node))
5308 if (tryShrinkShlLogicImm(
Node))
5312 if (tryVPTERNLOG(
Node))
5327 if (!CurDAG->shouldOptForSize())
5331 if (NVT != MVT::i8 && NVT != MVT::i16 && NVT != MVT::i32 && NVT != MVT::i64)
5337 auto *Cst = dyn_cast<ConstantSDNode>(N1);
5341 int64_t Val = Cst->getSExtValue();
5345 if (!isInt<8>(Val) && !isInt<32>(Val))
5349 if (Opcode ==
ISD::ADD && (Val == 1 || Val == -1))
5353 if (!shouldAvoidImmediateInstFormsForSize(N1.
getNode()))
5357 unsigned ROpc, MOpc;
5466 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5467 if (tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5469 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5470 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5474 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N0)->getMemOperand()});
5476 CurDAG->RemoveDeadNode(
Node);
5481 CurDAG->SelectNodeTo(
Node, ROpc, NVT, MVT::i32, N0, N1);
5494 unsigned LoReg, ROpc, MOpc;
5499 ROpc = Opcode ==
X86ISD::SMUL ? X86::IMUL8r : X86::MUL8r;
5500 MOpc = Opcode ==
X86ISD::SMUL ? X86::IMUL8m : X86::MUL8m;
5519 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5520 bool FoldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5523 FoldedLoad = tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5528 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5537 VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5539 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32, MVT::Other);
5543 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5548 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5554 VTs = CurDAG->getVTList(NVT, MVT::i32);
5556 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32);
5558 CNode = CurDAG->getMachineNode(ROpc, dl, VTs, {N1, InGlue});
5563 CurDAG->RemoveDeadNode(
Node);
5573 unsigned LoReg, HiReg;
5575 bool UseMULX = !IsSigned && Subtarget->hasBMI2();
5580 Opc = UseMULXHi ? X86::MULX32Hrr
5582 : IsSigned ?
X86::IMUL32r
5584 MOpc = UseMULXHi ? X86::MULX32Hrm
5586 : IsSigned ?
X86::IMUL32m
5588 LoReg = UseMULX ? X86::EDX : X86::EAX;
5592 Opc = UseMULXHi ? X86::MULX64Hrr
5594 : IsSigned ?
X86::IMUL64r
5596 MOpc = UseMULXHi ? X86::MULX64Hrm
5598 : IsSigned ?
X86::IMUL64m
5600 LoReg = UseMULX ? X86::RDX : X86::RAX;
5605 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5606 bool foldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5609 foldedLoad = tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5614 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5623 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
5624 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5627 }
else if (UseMULX) {
5628 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other);
5629 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5634 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5635 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5641 ReplaceUses(N1.
getValue(1), Chain);
5643 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5645 SDValue Ops[] = { N1, InGlue };
5647 SDVTList VTs = CurDAG->getVTList(NVT);
5648 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5650 }
else if (UseMULX) {
5651 SDVTList VTs = CurDAG->getVTList(NVT, NVT);
5652 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5656 SDVTList VTs = CurDAG->getVTList(MVT::Glue);
5657 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5665 assert(LoReg &&
"Register for low half is not defined!");
5666 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg,
5677 assert(HiReg &&
"Register for high half is not defined!");
5678 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg,
5687 CurDAG->RemoveDeadNode(
Node);
5696 unsigned ROpc, MOpc;
5701 case MVT::i8: ROpc = X86::DIV8r; MOpc = X86::DIV8m;
break;
5702 case MVT::i16: ROpc = X86::DIV16r; MOpc = X86::DIV16m;
break;
5703 case MVT::i32: ROpc = X86::DIV32r; MOpc = X86::DIV32m;
break;
5704 case MVT::i64: ROpc = X86::DIV64r; MOpc = X86::DIV64m;
break;
5709 case MVT::i8: ROpc = X86::IDIV8r; MOpc = X86::IDIV8m;
break;
5710 case MVT::i16: ROpc = X86::IDIV16r; MOpc = X86::IDIV16m;
break;
5711 case MVT::i32: ROpc = X86::IDIV32r; MOpc = X86::IDIV32m;
break;
5712 case MVT::i64: ROpc = X86::IDIV64r; MOpc = X86::IDIV64m;
break;
5716 unsigned LoReg, HiReg, ClrReg;
5717 unsigned SExtOpcode;
5721 LoReg = X86::AL; ClrReg = HiReg = X86::AH;
5725 LoReg = X86::AX; HiReg = X86::DX;
5727 SExtOpcode = X86::CWD;
5730 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX;
5731 SExtOpcode = X86::CDQ;
5734 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX;
5735 SExtOpcode = X86::CQO;
5739 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5740 bool foldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5741 bool signBitIsZero = CurDAG->SignBitIsZero(N0);
5744 if (NVT == MVT::i8) {
5747 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain;
5749 if (tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5751 unsigned Opc = (
isSigned && !signBitIsZero) ? X86::MOVSX16rm8
5753 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, MVT::Other, Ops);
5755 ReplaceUses(N0.
getValue(1), Chain);
5757 CurDAG->setNodeMemRefs(Move, {cast<LoadSDNode>(N0)->getMemOperand()});
5759 unsigned Opc = (
isSigned && !signBitIsZero) ? X86::MOVSX16rr8
5761 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, N0);
5762 Chain = CurDAG->getEntryNode();
5764 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX,
SDValue(Move, 0),
5769 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
5770 LoReg, N0,
SDValue()).getValue(1);
5774 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InGlue),0);
5777 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
5779 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
5783 SDValue(CurDAG->getMachineNode(
5784 TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode,
5785 CurDAG->getTargetConstant(X86::sub_16bit, dl,
5793 SDValue(CurDAG->getMachineNode(
5794 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
5795 CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode,
5796 CurDAG->getTargetConstant(X86::sub_32bit, dl,
5804 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg,
5805 ClrNode, InGlue).getValue(1);
5813 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops);
5818 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5821 SDValue(CurDAG->getMachineNode(ROpc, dl, MVT::Glue, N1, InGlue), 0);
5831 if (HiReg == X86::AH && !
SDValue(
Node, 1).use_empty()) {
5832 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8);
5833 unsigned AHExtOpcode =
5834 isSigned ? X86::MOVSX32rr8_NOREX : X86::MOVZX32rr8_NOREX;
5836 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32,
5837 MVT::Glue, AHCopy, InGlue);
5842 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result);
5850 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5851 LoReg, NVT, InGlue);
5852 InGlue =
Result.getValue(2);
5859 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5860 HiReg, NVT, InGlue);
5861 InGlue =
Result.getValue(2);
5866 CurDAG->RemoveDeadNode(
Node);
5875 SDValue N0 =
Node->getOperand(IsStrictCmp ? 1 : 0);
5876 SDValue N1 =
Node->getOperand(IsStrictCmp ? 2 : 1);
5882 if (Subtarget->canUseCMOV())
5891 Opc = IsSignaling ? X86::COM_Fpr32 : X86::UCOM_Fpr32;
5894 Opc = IsSignaling ? X86::COM_Fpr64 : X86::UCOM_Fpr64;
5897 Opc = IsSignaling ? X86::COM_Fpr80 : X86::UCOM_Fpr80;
5902 IsStrictCmp ?
Node->getOperand(0) : CurDAG->getEntryNode();
5905 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5906 Chain =
SDValue(CurDAG->getMachineNode(Opc, dl, VTs, {N0, N1, Chain}), 0);
5909 Glue =
SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N0, N1), 0);
5914 SDValue(CurDAG->getMachineNode(X86::FNSTSW16r, dl, MVT::i16, Glue), 0);
5918 CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, MVT::i8, FNSTSW);
5922 assert(Subtarget->canUseLAHFSAHF() &&
5923 "Target doesn't support SAHF or FCOMI?");
5924 SDValue AH = CurDAG->getCopyToReg(Chain, dl, X86::AH, Extract,
SDValue());
5927 CurDAG->getMachineNode(X86::SAHF, dl, MVT::i32, AH.
getValue(1)), 0);
5933 CurDAG->RemoveDeadNode(
Node);
5954 unsigned TestOpc = CmpVT == MVT::i64 ? X86::TEST64rr
5957 NewNode = CurDAG->getMachineNode(TestOpc, dl, MVT::i32, BEXTR, BEXTR);
5959 CurDAG->RemoveDeadNode(
Node);
5973 auto *MaskC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
5991 unsigned TestOpcode;
5999 if (LeadingZeros == 0 && SavesBytes) {
6004 ShiftAmt = TrailingZeros;
6006 TestOpcode = X86::TEST64rr;
6007 }
else if (TrailingZeros == 0 && SavesBytes) {
6012 ShiftAmt = LeadingZeros;
6014 TestOpcode = X86::TEST64rr;
6015 }
else if (MaskC->hasOneUse() && !isInt<32>(Mask)) {
6018 unsigned PopCount = 64 - LeadingZeros - TrailingZeros;
6019 if (PopCount == 8) {
6021 ShiftAmt = TrailingZeros;
6022 SubRegIdx = X86::sub_8bit;
6024 TestOpcode = X86::TEST8rr;
6025 }
else if (PopCount == 16) {
6027 ShiftAmt = TrailingZeros;
6028 SubRegIdx = X86::sub_16bit;
6029 SubRegVT = MVT::i16;
6030 TestOpcode = X86::TEST16rr;
6031 }
else if (PopCount == 32) {
6033 ShiftAmt = TrailingZeros;
6034 SubRegIdx = X86::sub_32bit;
6035 SubRegVT = MVT::i32;
6036 TestOpcode = X86::TEST32rr;
6040 SDValue ShiftC = CurDAG->getTargetConstant(ShiftAmt, dl, MVT::i64);
6042 CurDAG->getMachineNode(ShiftOpcode, dl, MVT::i64, MVT::i32,
6045 if (SubRegIdx != 0) {
6047 CurDAG->getTargetExtractSubreg(SubRegIdx, dl, SubRegVT, Shift);
6050 CurDAG->getMachineNode(TestOpcode, dl, MVT::i32, Shift, Shift);
6058 unsigned ROpc, MOpc;
6065 if (isUInt<8>(Mask) &&
6066 (!(Mask & 0x80) || CmpVT == MVT::i8 ||
6070 SubRegOp = X86::sub_8bit;
6071 ROpc = X86::TEST8ri;
6072 MOpc = X86::TEST8mi;
6073 }
else if (OptForMinSize && isUInt<16>(Mask) &&
6074 (!(Mask & 0x8000) || CmpVT == MVT::i16 ||
6081 SubRegOp = X86::sub_16bit;
6082 ROpc = X86::TEST16ri;
6083 MOpc = X86::TEST16mi;
6084 }
else if (isUInt<32>(Mask) && N0.
getValueType() != MVT::i16 &&
6085 ((!(Mask & 0x80000000) &&
6088 (CmpVT != MVT::i16 || !(Mask & 0x8000))) ||
6089 CmpVT == MVT::i32 ||
6097 SubRegOp = X86::sub_32bit;
6098 ROpc = X86::TEST32ri;
6099 MOpc = X86::TEST32mi;
6105 SDValue Imm = CurDAG->getTargetConstant(Mask, dl, VT);
6110 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
6111 if (tryFoldLoad(
Node, N0.
getNode(), Reg, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
6113 if (!LoadN->isSimple()) {
6115 if ((MOpc == X86::TEST8mi && NumVolBits != 8) ||
6116 (MOpc == X86::TEST16mi && NumVolBits != 16) ||
6117 (MOpc == X86::TEST32mi && NumVolBits != 32))
6121 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
6122 Reg.getOperand(0) };
6123 NewNode = CurDAG->getMachineNode(MOpc, dl, MVT::i32, MVT::Other, Ops);
6125 ReplaceUses(
Reg.getValue(1),
SDValue(NewNode, 1));
6127 CurDAG->setNodeMemRefs(NewNode,
6128 {cast<LoadSDNode>(Reg)->getMemOperand()});
6132 Reg = CurDAG->getTargetExtractSubreg(SubRegOp, dl, VT, Reg);
6134 NewNode = CurDAG->getMachineNode(ROpc, dl, MVT::i32, Reg, Imm);
6137 ReplaceNode(
Node, NewNode);
6143 if (!Subtarget->hasSSE42())
6149 bool MayFoldLoad = !NeedIndex || !NeedMask;
6154 Subtarget->hasAVX() ? X86::VPCMPISTRMrri : X86::PCMPISTRMrri;
6156 Subtarget->hasAVX() ? X86::VPCMPISTRMrmi : X86::PCMPISTRMrmi;
6157 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8,
Node);
6160 if (NeedIndex || !NeedMask) {
6162 Subtarget->hasAVX() ? X86::VPCMPISTRIrri : X86::PCMPISTRIrri;
6164 Subtarget->hasAVX() ? X86::VPCMPISTRIrmi : X86::PCMPISTRIrmi;
6165 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32,
Node);
6171 CurDAG->RemoveDeadNode(
Node);
6175 if (!Subtarget->hasSSE42())
6179 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX,
6180 Node->getOperand(1),
6182 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EDX,
6183 Node->getOperand(3), InGlue).getValue(1);
6188 bool MayFoldLoad = !NeedIndex || !NeedMask;
6193 Subtarget->hasAVX() ? X86::VPCMPESTRMrri : X86::PCMPESTRMrri;
6195 Subtarget->hasAVX() ? X86::VPCMPESTRMrmi : X86::PCMPESTRMrmi;
6197 emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8,
Node, InGlue);
6200 if (NeedIndex || !NeedMask) {
6202 Subtarget->hasAVX() ? X86::VPCMPESTRIrri : X86::PCMPESTRIrri;
6204 Subtarget->hasAVX() ? X86::VPCMPESTRIrmi : X86::PCMPESTRIrmi;
6205 CNode = emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32,
Node, InGlue);
6210 CurDAG->RemoveDeadNode(
Node);
6222 if (foldLoadStoreIntoMemOperand(
Node))
6227 MVT VT =
Node->getSimpleValueType(0);
6229 if (Subtarget->hasSBBDepBreaking()) {
6234 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
6239 unsigned Opc = VT == MVT::i64 ? X86::SETB_C64r : X86::SETB_C32r;
6240 MVT SetVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
6242 CurDAG->getMachineNode(Opc, dl, SetVT, EFLAGS, EFLAGS.
getValue(1)),
6251 if (VT == MVT::i8 || VT == MVT::i16) {
6252 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6253 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6257 CurDAG->RemoveDeadNode(
Node);
6271 MVT VT =
Node->getSimpleValueType(0);
6272 if (VT == MVT::i8 || VT == MVT::i16) {
6273 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6274 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6279 CurDAG->RemoveDeadNode(
Node);
6285 auto *Mgt = cast<X86MaskedGatherSDNode>(
Node);
6286 SDValue IndexOp = Mgt->getIndex();
6289 MVT ValueVT =
Node->getSimpleValueType(0);
6290 MVT MaskVT =
Mask.getSimpleValueType();
6307 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6308 Opc = IsFP ? X86::VGATHERDPSZ128rm : X86::VPGATHERDDZ128rm;
6309 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6310 Opc = IsFP ? X86::VGATHERDPSZ256rm : X86::VPGATHERDDZ256rm;
6311 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6312 Opc = IsFP ? X86::VGATHERDPSZrm : X86::VPGATHERDDZrm;
6313 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6314 Opc = IsFP ? X86::VGATHERDPDZ128rm : X86::VPGATHERDQZ128rm;
6315 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6316 Opc = IsFP ? X86::VGATHERDPDZ256rm : X86::VPGATHERDQZ256rm;
6317 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6318 Opc = IsFP ? X86::VGATHERDPDZrm : X86::VPGATHERDQZrm;
6319 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6320 Opc = IsFP ? X86::VGATHERQPSZ128rm : X86::VPGATHERQDZ128rm;
6321 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6322 Opc = IsFP ? X86::VGATHERQPSZ256rm : X86::VPGATHERQDZ256rm;
6323 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6324 Opc = IsFP ? X86::VGATHERQPSZrm : X86::VPGATHERQDZrm;
6325 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6326 Opc = IsFP ? X86::VGATHERQPDZ128rm : X86::VPGATHERQQZ128rm;
6327 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6328 Opc = IsFP ? X86::VGATHERQPDZ256rm : X86::VPGATHERQQZ256rm;
6329 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6330 Opc = IsFP ? X86::VGATHERQPDZrm : X86::VPGATHERQQZrm;
6332 assert(
EVT(MaskVT) ==
EVT(ValueVT).changeVectorElementTypeToInteger() &&
6333 "Unexpected mask VT!");
6334 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6335 Opc = IsFP ? X86::VGATHERDPSrm : X86::VPGATHERDDrm;
6336 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6337 Opc = IsFP ? X86::VGATHERDPSYrm : X86::VPGATHERDDYrm;
6338 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6339 Opc = IsFP ? X86::VGATHERDPDrm : X86::VPGATHERDQrm;
6340 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6341 Opc = IsFP ? X86::VGATHERDPDYrm : X86::VPGATHERDQYrm;
6342 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6343 Opc = IsFP ? X86::VGATHERQPSrm : X86::VPGATHERQDrm;
6344 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6345 Opc = IsFP ? X86::VGATHERQPSYrm : X86::VPGATHERQDYrm;
6346 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6347 Opc = IsFP ? X86::VGATHERQPDrm : X86::VPGATHERQQrm;
6348 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6349 Opc = IsFP ? X86::VGATHERQPDYrm : X86::VPGATHERQQYrm;
6356 if (!selectVectorAddr(Mgt, Mgt->getBasePtr(), IndexOp, Mgt->getScale(),
6360 SDValue PassThru = Mgt->getPassThru();
6361 SDValue Chain = Mgt->getChain();
6363 SDVTList VTs = CurDAG->getVTList(ValueVT, MaskVT, MVT::Other);
6368 Index, Disp, Segment, Chain};
6369 NewNode = CurDAG->getMachineNode(Opc,
SDLoc(dl), VTs, Ops);
6372 Disp, Segment,
Mask, Chain};
6373 NewNode = CurDAG->getMachineNode(Opc,
SDLoc(dl), VTs, Ops);
6375 CurDAG->setNodeMemRefs(NewNode, {Mgt->getMemOperand()});
6378 CurDAG->RemoveDeadNode(
Node);
6382 auto *Sc = cast<X86MaskedScatterSDNode>(
Node);
6384 SDValue IndexOp = Sc->getIndex();
6386 MVT ValueVT =
Value.getSimpleValueType();
6401 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6402 Opc = IsFP ? X86::VSCATTERDPSZ128mr : X86::VPSCATTERDDZ128mr;
6403 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6404 Opc = IsFP ? X86::VSCATTERDPSZ256mr : X86::VPSCATTERDDZ256mr;
6405 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6406 Opc = IsFP ? X86::VSCATTERDPSZmr : X86::VPSCATTERDDZmr;
6407 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6408 Opc = IsFP ? X86::VSCATTERDPDZ128mr : X86::VPSCATTERDQZ128mr;
6409 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6410 Opc = IsFP ? X86::VSCATTERDPDZ256mr : X86::VPSCATTERDQZ256mr;
6411 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6412 Opc = IsFP ? X86::VSCATTERDPDZmr : X86::VPSCATTERDQZmr;
6413 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6414 Opc = IsFP ? X86::VSCATTERQPSZ128mr : X86::VPSCATTERQDZ128mr;
6415 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6416 Opc = IsFP ? X86::VSCATTERQPSZ256mr : X86::VPSCATTERQDZ256mr;
6417 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6418 Opc = IsFP ? X86::VSCATTERQPSZmr : X86::VPSCATTERQDZmr;
6419 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6420 Opc = IsFP ? X86::VSCATTERQPDZ128mr : X86::VPSCATTERQQZ128mr;
6421 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6422 Opc = IsFP ? X86::VSCATTERQPDZ256mr : X86::VPSCATTERQQZ256mr;
6423 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6424 Opc = IsFP ? X86::VSCATTERQPDZmr : X86::VPSCATTERQQZmr;
6429 if (!selectVectorAddr(Sc, Sc->getBasePtr(), IndexOp, Sc->getScale(),
6434 SDValue Chain = Sc->getChain();
6436 SDVTList VTs = CurDAG->getVTList(
Mask.getValueType(), MVT::Other);
6440 CurDAG->setNodeMemRefs(NewNode, {Sc->getMemOperand()});
6442 CurDAG->RemoveDeadNode(
Node);
6448 cast<SrcValueSDNode>(
Node->getOperand(1))->getValue());
6450 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6452 TargetOpcode::PREALLOCATED_SETUP, dl, MVT::Other, CallIdValue, Chain);
6454 CurDAG->RemoveDeadNode(
Node);
6460 cast<SrcValueSDNode>(
Node->getOperand(1))->getValue());
6462 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6465 Ops[0] = CallIdValue;
6469 TargetOpcode::PREALLOCATED_ARG, dl,
6470 CurDAG->getVTList(TLI->
getPointerTy(CurDAG->getDataLayout()),
6475 CurDAG->RemoveDeadNode(
Node);
6482 if (!Subtarget->hasWIDEKL())
6486 switch (
Node->getOpcode()) {
6490 Opcode = X86::AESENCWIDE128KL;
6493 Opcode = X86::AESDECWIDE128KL;
6496 Opcode = X86::AESENCWIDE256KL;
6499 Opcode = X86::AESDECWIDE256KL;
6510 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(2),
6512 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(3),
6514 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM2,
Node->getOperand(4),
6516 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM3,
Node->getOperand(5),
6518 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM4,
Node->getOperand(6),
6520 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM5,
Node->getOperand(7),
6522 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM6,
Node->getOperand(8),
6524 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM7,
Node->getOperand(9),
6528 Opcode, dl,
Node->getVTList(),
6529 {Base, Scale, Index, Disp, Segment, Chain, Chain.getValue(1)});
6530 CurDAG->setNodeMemRefs(Res, cast<MemSDNode>(
Node)->getMemOperand());
6531 ReplaceNode(
Node, Res);
6539bool X86DAGToDAGISel::SelectInlineAsmMemoryOperand(
6541 std::vector<SDValue> &OutOps) {
6542 SDValue Op0, Op1, Op2, Op3, Op4;
6543 switch (ConstraintID) {
6546 case InlineAsm::ConstraintCode::o:
6547 case InlineAsm::ConstraintCode::v:
6548 case InlineAsm::ConstraintCode::m:
6549 case InlineAsm::ConstraintCode::X:
6550 case InlineAsm::ConstraintCode::p:
6551 if (!selectAddr(
nullptr,
Op, Op0, Op1, Op2, Op3, Op4))
6556 OutOps.push_back(Op0);
6557 OutOps.push_back(Op1);
6558 OutOps.push_back(Op2);
6559 OutOps.push_back(Op3);
6560 OutOps.push_back(Op4);
6568 return new X86DAGToDAGISel(
TM, OptLevel);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool isRIPRelative(const MCInst &MI, const MCInstrInfo &MCII)
Check if the instruction uses RIP relative addressing.
static bool isLegalMaskCompare(SDNode *N, const X86Subtarget *Subtarget)
static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, unsigned LoadOpNo, LoadSDNode *&LoadNode, SDValue &InputChain)
Check whether or not the chain ending in StoreNode is suitable for doing the {load; op; store} to mod...
#define GET_EGPR_IF_ENABLED(OPC)
static bool needBWI(MVT VT)
static unsigned getVPTESTMOpc(MVT TestVT, bool IsTestN, bool FoldedLoad, bool FoldedBCast, bool Masked)
static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM, const X86Subtarget &Subtarget)
static bool mayUseCarryFlag(X86::CondCode CC)
static cl::opt< bool > EnablePromoteAnyextLoad("x86-promote-anyext-load", cl::init(true), cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden)
cl::opt< bool > IndirectBranchTracking
static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, SDValue Call, SDValue OrigChain)
Replace the original chain operand of the call with load's chain operand and move load below the call...
static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N)
#define GET_ND_IF_ENABLED(OPC)
#define VPTESTM_BROADCAST_CASES(SUFFIX)
static cl::opt< bool > AndImmShrink("x86-and-imm-shrink", cl::init(true), cl::desc("Enable setting constant bits to reduce size of mask immediates"), cl::Hidden)
static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, X86ISelAddressMode &AM)
#define VPTESTM_FULL_CASES(SUFFIX)
static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq)
Return true if call address is a load and it can be moved below CALLSEQ_START and the chains leading ...
static bool isDispSafeForFrameIndex(int64_t Val)
static bool isEndbrImm64(uint64_t Imm)
#define GET_ND_IF_ENABLED(OPC)
DEMANGLE_DUMP_METHOD void dump() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
APInt trunc(unsigned width) const
Truncate to new width.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countl_zero() const
The APInt version of std::countl_zero.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
bool isOne() const
Determine if this is a value of 1.
unsigned countr_one() const
Count the number of trailing one bits.
The address of a basic block.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
std::optional< ConstantRange > getAbsoluteSymbolRange() const
If this is an absolute symbol reference, returns the range of the symbol, otherwise returns std::null...
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
Describe properties that are true of each instruction in the target description file.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
bool is128BitVector() const
Return true if this is a 128-bit vector type.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool is512BitVector() const
Return true if this is a 512-bit vector type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const Module * getModule() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isNonTemporal() const
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
static int getUninvalidatedNodeId(SDNode *N)
virtual void emitFunctionEntryCode()
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
static constexpr unsigned MaxRecursionDepth
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
static Type * getVoidTy(LLVMContext &C)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void dump() const
Support for debugging, callable in GDB: V->dump()
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
size_t getPreallocatedIdForCallSite(const Value *CS)
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BRIND
BRIND - Indirect branch.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
@ GlobalBaseReg
The result of the mflr at function entry, used for PIC code.
@ X86
Windows x64, Windows Itanium (IA-64)
Reg
All possible values of the reg field in the ModR/M byte.
@ MO_NO_FLAG
MO_NO_FLAG - No flag for the operand.
@ EVEX
EVEX - Specifies that this instruction use EVEX form which provides syntax support up to 32 512-bit r...
@ VEX
VEX - encoding using 0xC4/0xC5.
@ XOP
XOP - Opcode prefix used by XOP instructions.
@ FST
This instruction implements a truncating store from FP stack slots.
@ CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ CMP
X86 compare and logical compare instructions.
@ BLENDV
Dynamic (non-constant condition) vector blend where only the sign bits of the condition elements are ...
@ STRICT_FCMP
X86 strict FP compare instructions.
@ STRICT_CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ NT_BRIND
BRIND node with NoTrack prefix.
@ FSETCCM
X86 FP SETCC, similar to above, but with output as an i1 mask and and a version with SAE.
@ FXOR
Bitwise logical XOR of floating point values.
@ BRCOND
X86 conditional branches.
@ CALL
These operations represent an abstract X86 call instruction, which includes a bunch of information.
@ FANDN
Bitwise logical ANDNOT of floating point values.
@ GlobalBaseReg
On Darwin, this node represents the result of the popl at function entry, used for PIC code.
@ FLD
This instruction implements an extending load to FP stack slots.
@ TC_RETURN
Tail call return.
@ FOR
Bitwise logical OR of floating point values.
@ Wrapper
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress,...
@ ANDNP
Bitwise Logical AND NOT of Packed FP values.
@ FAND
Bitwise logical AND of floating point values.
@ CMOV
X86 conditional moves.
@ WrapperRIP
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
int getCondSrcNoFromDesc(const MCInstrDesc &MCID)
Return the source operand # for condition code by MCID.
bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into some other x86 instruction as a memory oper...
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement)
Returns true of the given offset can be fit into displacement field of the instruction.
bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs)
If Op is a constant whose elements are all the same constant or undefined, return true and return the...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
int popcount(T Value) noexcept
Count the number of set bits in a value.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
unsigned M1(unsigned Val)
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
FunctionPass * createX86ISelDag(X86TargetMachine &TM, CodeGenOptLevel OptLevel)
This pass converts a legalized DAG into a X86-specific DAG, ready for instruction scheduling.
CodeGenOptLevel
Code generation optimization level.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.