22#include "llvm/Config/llvm-config.h"
27#include "llvm/IR/IntrinsicsX86.h"
37#define DEBUG_TYPE "x86-isel"
38#define PASS_NAME "X86 DAG->DAG Instruction Selection"
40STATISTIC(NumLoadMoved,
"Number of loads moved below TokenFactor");
43 cl::desc(
"Enable setting constant bits to reduce size of mask immediates"),
47 "x86-promote-anyext-load",
cl::init(
true),
59 struct X86ISelAddressMode {
67 int Base_FrameIndex = 0;
76 const char *ES =
nullptr;
81 bool NegateIndex =
false;
83 X86ISelAddressMode() =
default;
85 bool hasSymbolicDisplacement()
const {
86 return GV !=
nullptr ||
CP !=
nullptr || ES !=
nullptr ||
87 MCSym !=
nullptr ||
JT != -1 || BlockAddr !=
nullptr;
90 bool hasBaseOrIndexReg()
const {
97 if (
BaseType != RegBase)
return false;
99 dyn_cast_or_null<RegisterSDNode>(Base_Reg.
getNode()))
100 return RegNode->getReg() == X86::RIP;
109#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
111 dbgs() <<
"X86ISelAddressMode " <<
this <<
'\n';
112 dbgs() <<
"Base_Reg ";
118 dbgs() <<
" Base.FrameIndex " << Base_FrameIndex <<
'\n';
119 dbgs() <<
" Scale " << Scale <<
'\n'
127 dbgs() <<
" Disp " << Disp <<
'\n'
149 dbgs() <<
" JT" <<
JT <<
" Align" << Alignment.
value() <<
'\n';
169 bool IndirectTlsSegRefs;
174 X86DAGToDAGISel() =
delete;
178 OptForMinSize(
false), IndirectTlsSegRefs(
false) {}
184 "indirect-tls-seg-refs");
189 "OptForMinSize implies OptForSize");
203#include "X86GenDAGISel.inc"
208 bool foldOffsetIntoAddress(
uint64_t Offset, X86ISelAddressMode &AM);
209 bool matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
210 bool AllowSegmentRegForX32 =
false);
211 bool matchWrapper(
SDValue N, X86ISelAddressMode &AM);
212 bool matchAddress(
SDValue N, X86ISelAddressMode &AM);
213 bool matchVectorAddress(
SDValue N, X86ISelAddressMode &AM);
214 bool matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
unsigned Depth);
217 bool matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
219 bool matchVectorAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
221 bool matchAddressBase(
SDValue N, X86ISelAddressMode &AM);
250 return tryFoldLoad(
P,
P,
N,
Base, Scale,
Index, Disp, Segment);
258 bool isProfitableToFormMaskedOp(
SDNode *
N)
const;
263 std::vector<SDValue> &OutOps)
override;
265 void emitSpecialCodeForMain();
267 inline void getAddressOperands(X86ISelAddressMode &AM,
const SDLoc &
DL,
271 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
272 Base = CurDAG->getTargetFrameIndex(
273 AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout()));
274 else if (AM.Base_Reg.getNode())
277 Base = CurDAG->getRegister(0, VT);
279 Scale = getI8Imm(AM.Scale,
DL);
282 if (AM.NegateIndex) {
283 unsigned NegOpc = VT == MVT::i64 ? X86::NEG64r : X86::NEG32r;
289 if (AM.IndexReg.getNode())
292 Index = CurDAG->getRegister(0, VT);
297 Disp = CurDAG->getTargetGlobalAddress(AM.GV,
SDLoc(),
301 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment,
302 AM.Disp, AM.SymbolFlags);
304 assert(!AM.Disp &&
"Non-zero displacement is ignored with ES.");
305 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
306 }
else if (AM.MCSym) {
307 assert(!AM.Disp &&
"Non-zero displacement is ignored with MCSym.");
308 assert(AM.SymbolFlags == 0 &&
"oo");
309 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32);
310 }
else if (AM.JT != -1) {
311 assert(!AM.Disp &&
"Non-zero displacement is ignored with JT.");
312 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
313 }
else if (AM.BlockAddr)
314 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
317 Disp = CurDAG->getTargetConstant(AM.Disp,
DL, MVT::i32);
319 if (AM.Segment.getNode())
320 Segment = AM.Segment;
322 Segment = CurDAG->getRegister(0, MVT::i16);
331 bool shouldAvoidImmediateInstFormsForSize(
SDNode *
N)
const {
337 if (!CurDAG->shouldOptForSize())
347 if (
User->isMachineOpcode()) {
370 auto *
C = dyn_cast<ConstantSDNode>(
N);
371 if (
C && isInt<8>(
C->getSExtValue()))
391 (RegNode = dyn_cast_or_null<RegisterSDNode>(
393 if ((RegNode->
getReg() == X86::ESP) ||
394 (RegNode->
getReg() == X86::RSP))
403 return (UseCount > 1);
408 return CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
413 return CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
418 return CurDAG->getTargetConstant(Imm,
DL, MVT::i64);
423 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
425 MVT VecVT =
N->getOperand(0).getSimpleValueType();
431 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
433 MVT VecVT =
N->getSimpleValueType(0);
437 SDValue getPermuteVINSERTCommutedImmediate(
SDNode *
N,
unsigned VecWidth,
439 assert(VecWidth == 128 &&
"Unexpected vector width");
441 MVT VecVT =
N->getSimpleValueType(0);
443 assert((InsertIdx == 0 || InsertIdx == 1) &&
"Bad insertf128 index");
446 return getI8Imm(InsertIdx ? 0x02 : 0x30,
DL);
451 MVT VT =
N->getSimpleValueType(0);
454 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
456 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
457 if (VT == MVT::i64) {
459 CurDAG->getMachineNode(
460 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
461 CurDAG->getTargetConstant(0, dl, MVT::i64), Zero,
462 CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)),
467 unsigned Opcode =
N->getOpcode();
469 "Unexpected opcode for SBB materialization");
472 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
473 N->getOperand(FlagOpIndex),
SDValue());
477 unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr;
478 MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
479 VTs = CurDAG->getVTList(SBBVT, MVT::i32);
481 CurDAG->getMachineNode(Opc, dl, VTs,
482 {Zero, Zero, EFLAGS, EFLAGS.getValue(1)}),
488 bool isUnneededShiftMask(
SDNode *
N,
unsigned Width)
const {
490 const APInt &Val = cast<ConstantSDNode>(
N->getOperand(1))->getAPIntValue();
495 APInt Mask = Val | CurDAG->computeKnownBits(
N->getOperand(0)).Zero;
502 SDNode *getGlobalBaseReg();
513 return Subtarget->getInstrInfo();
526 bool isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const;
530 if (!
N->isNonTemporal())
533 unsigned StoreSize =
N->getMemoryVT().getStoreSize();
535 if (
N->getAlign().value() < StoreSize)
544 return Subtarget->hasSSE41();
546 return Subtarget->hasAVX2();
548 return Subtarget->hasAVX512();
552 bool foldLoadStoreIntoMemOperand(
SDNode *
Node);
555 bool shrinkAndImmediate(
SDNode *
N);
556 bool isMaskZeroExtended(
SDNode *
N)
const;
557 bool tryShiftAmountMod(
SDNode *
N);
558 bool tryShrinkShlLogicImm(
SDNode *
N);
564 bool tryMatchBitSelect(
SDNode *
N);
566 MachineSDNode *emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
568 MachineSDNode *emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
572 bool tryOptimizeRem8Extend(
SDNode *
N);
574 bool onlyUsesZeroFlag(
SDValue Flags)
const;
575 bool hasNoSignFlagUses(
SDValue Flags)
const;
576 bool hasNoCarryFlagUses(
SDValue Flags)
const;
580char X86DAGToDAGISel::ID = 0;
587 unsigned Opcode =
N->getOpcode();
594 EVT OpVT =
N->getOperand(0).getValueType();
598 OpVT =
N->getOperand(1).getValueType();
600 return Subtarget->hasVLX();
614bool X86DAGToDAGISel::isMaskZeroExtended(
SDNode *
N)
const {
627 if (OptLevel == CodeGenOptLevel::None)
637 if (useNonTemporalLoad(cast<LoadSDNode>(
N)))
642 switch (
U->getOpcode()) {
668 if (
auto *Imm = dyn_cast<ConstantSDNode>(Op1)) {
669 if (
Imm->getAPIntValue().isSignedIntN(8))
678 Imm->getAPIntValue().getBitWidth() == 64 &&
679 Imm->getAPIntValue().isIntN(32))
686 (
Imm->getAPIntValue() == UINT8_MAX ||
687 Imm->getAPIntValue() == UINT16_MAX ||
688 Imm->getAPIntValue() == UINT32_MAX))
694 (-
Imm->getAPIntValue()).isSignedIntN(8))
698 (-
Imm->getAPIntValue()).isSignedIntN(8) &&
699 hasNoCarryFlagUses(
SDValue(U, 1)))
724 if (
U->getOperand(0).getOpcode() ==
ISD::SHL &&
728 if (
U->getOperand(1).getOpcode() ==
ISD::SHL &&
736 auto *
C = dyn_cast<ConstantSDNode>(U0.
getOperand(0));
737 if (
C &&
C->getSExtValue() == -2)
742 auto *
C = dyn_cast<ConstantSDNode>(U1.
getOperand(0));
743 if (
C &&
C->getSExtValue() == -2)
757 if (isa<ConstantSDNode>(
U->getOperand(1)))
778bool X86DAGToDAGISel::isProfitableToFormMaskedOp(
SDNode *
N)
const {
781 "Unexpected opcode!");
786 return N->getOperand(1).hasOneUse();
795 if (Chain.
getNode() == Load.getNode())
799 "Unexpected chain operand");
813 Load.getOperand(1), Load.getOperand(2));
817 Ops.
append(Call->op_begin() + 1, Call->op_end());
831 if (Callee.getNode() == Chain.
getNode() || !Callee.hasOneUse())
833 auto *LD = dyn_cast<LoadSDNode>(Callee.getNode());
851 if (isa<MemSDNode>(Chain.
getNode()) &&
852 cast<MemSDNode>(Chain.
getNode())->writeMem())
858 Callee.getValue(1).hasOneUse())
866 if ((Imm & 0x00FFFFFF) != 0x0F1EFA)
869 uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64,
870 0x65, 0x66, 0x67, 0xf0, 0xf2};
873 uint8_t Byte = (Imm >> i) & 0xFF;
885 return (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v64i8);
888void X86DAGToDAGISel::PreprocessISelDAG() {
889 bool MadeChange =
false;
891 E = CurDAG->allnodes_end();
I !=
E; ) {
910 MVT VT =
N->getSimpleValueType(0);
911 int64_t
Imm = cast<ConstantSDNode>(
N)->getSExtValue();
912 int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB;
919 SDValue Complement = CurDAG->getConstant(~Imm, dl, VT,
false,
true);
920 Complement = CurDAG->getNOT(dl, Complement, VT);
922 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Complement);
932 if (
N->getOpcode() ==
X86ISD::AND && !
N->hasAnyUseOfValue(1)) {
934 N->getOperand(0),
N->getOperand(1));
936 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
960 auto mayPreventLoadFold = [&]() {
962 N->getOpcode() ==
ISD::ADD && Subtarget->hasAVX() &&
963 !
N->getOperand(1).hasOneUse();
966 N->getSimpleValueType(0).isVector() && !mayPreventLoadFold()) {
972 MVT VT =
N->getSimpleValueType(0);
980 CurDAG->getNode(NewOpcode,
DL, VT,
N->getOperand(0),
AllOnes);
982 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
989 switch (
N->getOpcode()) {
991 MVT VT =
N->getSimpleValueType(0);
993 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1000 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1003 CurDAG->getIntPtrConstant(
Index, dl));
1006 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1015 MVT VT =
N->getSimpleValueType(0);
1017 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1019 auto *MemNode = cast<MemSDNode>(
N);
1021 SDVTList VTs = CurDAG->getVTList(NarrowVT, MVT::Other);
1022 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()};
1023 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode(
1025 MemNode->getMemOperand());
1028 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1031 CurDAG->getIntPtrConstant(
Index, dl));
1035 CurDAG->ReplaceAllUsesWith(
N, To);
1046 auto *Ld = cast<LoadSDNode>(
N);
1047 MVT VT =
N->getSimpleValueType(0);
1055 SDValue Chain = Ld->getChain();
1057 auto *UserLd = dyn_cast<LoadSDNode>(
User);
1058 MVT UserVT =
User->getSimpleValueType(0);
1060 UserLd->getBasePtr() ==
Ptr && UserLd->getChain() == Chain &&
1061 !
User->hasAnyUseOfValue(1) &&
1075 CurDAG->getIntPtrConstant(0, dl));
1076 SDValue Res = CurDAG->getBitcast(VT, Extract);
1080 CurDAG->ReplaceAllUsesWith(
N, To);
1089 EVT EleVT =
N->getOperand(0).getValueType().getVectorElementType();
1090 if (EleVT == MVT::i1)
1093 assert(Subtarget->hasSSE41() &&
"Expected SSE4.1 support!");
1094 assert(
N->getValueType(0).getVectorElementType() != MVT::i16 &&
1095 "We can't replace VSELECT with BLENDV in vXi16!");
1097 if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(
N->getOperand(0)) ==
1100 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
1101 CurDAG->getTargetConstant(0xCA,
SDLoc(
N), MVT::i8));
1104 N->getOperand(0),
N->getOperand(1),
1108 CurDAG->ReplaceAllUsesWith(
N,
R.getNode());
1121 if (!
N->getSimpleValueType(0).isVector())
1125 switch (
N->getOpcode()) {
1135 if (
N->isStrictFPOpcode())
1137 CurDAG->getNode(NewOpc,
SDLoc(
N), {
N->getValueType(0), MVT::Other},
1138 {
N->getOperand(0),
N->getOperand(1)});
1141 CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1144 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1154 if (!
N->getValueType(0).isVector())
1158 switch (
N->getOpcode()) {
1164 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1165 N->getOperand(0),
N->getOperand(1));
1167 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1176 if (!
N->getValueType(0).isVector())
1180 if (
N->getOperand(0).getScalarValueSizeInBits() == 1) {
1182 "Unexpected opcode for mask vector!");
1190 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1193 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1213 switch (
N->getOpcode()) {
1229 bool IsStrict =
N->isStrictFPOpcode();
1233 {
N->getValueType(0), MVT::Other},
1234 {
N->getOperand(0),
N->getOperand(1),
1235 CurDAG->getTargetConstant(Imm, dl, MVT::i32)});
1239 CurDAG->getTargetConstant(Imm, dl, MVT::i32));
1241 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1252 MVT VT =
N->getSimpleValueType(0);
1253 if (VT.
isVector() || VT == MVT::f128)
1256 MVT VecVT = VT == MVT::f64 ? MVT::v2f64
1257 : VT == MVT::f32 ? MVT::v4f32
1267 if (Subtarget->hasSSE2()) {
1272 switch (
N->getOpcode()) {
1279 Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1);
1282 Res = CurDAG->getNode(
N->getOpcode(), dl, VecVT, Op0, Op1);
1285 CurDAG->getIntPtrConstant(0, dl));
1287 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1294 if (OptLevel != CodeGenOptLevel::None &&
1297 !Subtarget->useIndirectThunkCalls() &&
1298 ((
N->getOpcode() ==
X86ISD::CALL && !Subtarget->slowTwoMemOps()) ||
1300 (Subtarget->is64Bit() ||
1301 !getTargetMachine().isPositionIndependent())))) {
1340 switch (
N->getOpcode()) {
1345 MVT SrcVT =
N->getOperand(0).getSimpleValueType();
1346 MVT DstVT =
N->getSimpleValueType(0);
1358 if (SrcIsSSE && DstIsSSE)
1361 if (!SrcIsSSE && !DstIsSSE) {
1366 if (
N->getConstantOperandVal(1))
1374 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1375 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1383 CurDAG->getEntryNode(), dl,
N->getOperand(0), MemTmp, MPI, MemVT);
1385 MemTmp, MPI, MemVT);
1392 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Result);
1401 MVT SrcVT =
N->getOperand(1).getSimpleValueType();
1402 MVT DstVT =
N->getSimpleValueType(0);
1414 if (SrcIsSSE && DstIsSSE)
1417 if (!SrcIsSSE && !DstIsSSE) {
1422 if (
N->getConstantOperandVal(2))
1430 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1431 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1441 SDVTList VTs = CurDAG->getVTList(MVT::Other);
1442 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), MemTmp};
1446 if (
N->getFlags().hasNoFPExcept()) {
1448 Flags.setNoFPExcept(
true);
1449 Store->setFlags(Flags);
1452 assert(SrcVT == MemVT &&
"Unexpected VT!");
1453 Store = CurDAG->getStore(
N->getOperand(0), dl,
N->getOperand(1), MemTmp,
1458 SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other);
1460 Result = CurDAG->getMemIntrinsicNode(
1463 if (
N->getFlags().hasNoFPExcept()) {
1465 Flags.setNoFPExcept(
true);
1469 assert(DstVT == MemVT &&
"Unexpected VT!");
1470 Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI);
1478 CurDAG->ReplaceAllUsesWith(
N,
Result.getNode());
1492 CurDAG->RemoveDeadNodes();
1496bool X86DAGToDAGISel::tryOptimizeRem8Extend(
SDNode *
N) {
1497 unsigned Opc =
N->getMachineOpcode();
1498 if (Opc != X86::MOVZX32rr8 && Opc != X86::MOVSX32rr8 &&
1499 Opc != X86::MOVSX64rr8)
1511 unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX
1512 : X86::MOVSX32rr8_NOREX;
1517 if (Opc == X86::MOVSX64rr8) {
1522 ReplaceUses(
N, Extend);
1531void X86DAGToDAGISel::PostprocessISelDAG() {
1533 if (
TM.getOptLevel() == CodeGenOptLevel::None)
1538 bool MadeChange =
false;
1539 while (Position != CurDAG->allnodes_begin()) {
1542 if (
N->use_empty() || !
N->isMachineOpcode())
1545 if (tryOptimizeRem8Extend(
N)) {
1552 unsigned Opc =
N->getMachineOpcode();
1553 if ((Opc == X86::TEST8rr || Opc == X86::TEST16rr ||
1554 Opc == X86::TEST32rr || Opc == X86::TEST64rr) &&
1555 N->getOperand(0) ==
N->getOperand(1) &&
1556 N->getOperand(0)->hasNUsesOfValue(2,
N->getOperand(0).getResNo()) &&
1557 N->getOperand(0).isMachineOpcode()) {
1559 unsigned N0Opc =
And.getMachineOpcode();
1560 if ((N0Opc == X86::AND8rr || N0Opc == X86::AND16rr ||
1561 N0Opc == X86::AND32rr || N0Opc == X86::AND64rr) &&
1562 !
And->hasAnyUseOfValue(1)) {
1567 ReplaceUses(
N,
Test);
1571 if ((N0Opc == X86::AND8rm || N0Opc == X86::AND16rm ||
1572 N0Opc == X86::AND32rm || N0Opc == X86::AND64rm) &&
1573 !
And->hasAnyUseOfValue(1)) {
1576 case X86::AND8rm: NewOpc = X86::TEST8mr;
break;
1577 case X86::AND16rm: NewOpc = X86::TEST16mr;
break;
1578 case X86::AND32rm: NewOpc = X86::TEST32mr;
break;
1579 case X86::AND64rm: NewOpc = X86::TEST64mr;
break;
1589 And.getOperand(6) };
1591 MVT::i32, MVT::Other, Ops);
1592 CurDAG->setNodeMemRefs(
1593 Test, cast<MachineSDNode>(
And.getNode())->memoperands());
1605 if ((Opc == X86::KORTESTBrr || Opc == X86::KORTESTWrr ||
1606 Opc == X86::KORTESTDrr || Opc == X86::KORTESTQrr) &&
1607 N->getOperand(0) ==
N->getOperand(1) &&
1608 N->isOnlyUserOf(
N->getOperand(0).getNode()) &&
1609 N->getOperand(0).isMachineOpcode() &&
1612 unsigned N0Opc =
And.getMachineOpcode();
1615 if (N0Opc == X86::KANDBrr ||
1616 (N0Opc == X86::KANDWrr && Subtarget->hasDQI()) ||
1617 N0Opc == X86::KANDDrr || N0Opc == X86::KANDQrr) {
1621 case X86::KORTESTBrr: NewOpc = X86::KTESTBrr;
break;
1622 case X86::KORTESTWrr: NewOpc = X86::KTESTWrr;
break;
1623 case X86::KORTESTDrr: NewOpc = X86::KTESTDrr;
break;
1624 case X86::KORTESTQrr: NewOpc = X86::KTESTQrr;
break;
1630 ReplaceUses(
N, KTest);
1637 if (Opc != TargetOpcode::SUBREG_TO_REG)
1640 unsigned SubRegIdx =
N->getConstantOperandVal(2);
1641 if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm)
1652 case X86::VMOVAPDrr:
case X86::VMOVUPDrr:
1653 case X86::VMOVAPSrr:
case X86::VMOVUPSrr:
1654 case X86::VMOVDQArr:
case X86::VMOVDQUrr:
1655 case X86::VMOVAPDYrr:
case X86::VMOVUPDYrr:
1656 case X86::VMOVAPSYrr:
case X86::VMOVUPSYrr:
1657 case X86::VMOVDQAYrr:
case X86::VMOVDQUYrr:
1658 case X86::VMOVAPDZ128rr:
case X86::VMOVUPDZ128rr:
1659 case X86::VMOVAPSZ128rr:
case X86::VMOVUPSZ128rr:
1660 case X86::VMOVDQA32Z128rr:
case X86::VMOVDQU32Z128rr:
1661 case X86::VMOVDQA64Z128rr:
case X86::VMOVDQU64Z128rr:
1662 case X86::VMOVAPDZ256rr:
case X86::VMOVUPDZ256rr:
1663 case X86::VMOVAPSZ256rr:
case X86::VMOVUPSZ256rr:
1664 case X86::VMOVDQA32Z256rr:
case X86::VMOVDQU32Z256rr:
1665 case X86::VMOVDQA64Z256rr:
case X86::VMOVDQU64Z256rr:
1670 if (!
In.isMachineOpcode() ||
1671 In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END)
1684 CurDAG->UpdateNodeOperands(
N,
N->getOperand(0), In,
N->getOperand(2));
1689 CurDAG->RemoveDeadNodes();
1694void X86DAGToDAGISel::emitSpecialCodeForMain() {
1695 if (Subtarget->isTargetCygMing()) {
1697 auto &
DL = CurDAG->getDataLayout();
1700 CLI.setChain(CurDAG->getRoot())
1702 CurDAG->getExternalSymbol(
"__main", TLI->getPointerTy(
DL)),
1706 CurDAG->setRoot(
Result.second);
1710void X86DAGToDAGISel::emitFunctionEntryCode() {
1713 if (
F.hasExternalLinkage() &&
F.getName() ==
"main")
1714 emitSpecialCodeForMain();
1724 return isInt<31>(Val);
1728 X86ISelAddressMode &AM) {
1733 int64_t Val = AM.Disp +
Offset;
1736 if (Val != 0 && (AM.ES || AM.MCSym))
1740 if (Subtarget->is64Bit()) {
1743 AM.hasSymbolicDisplacement()))
1747 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
1766 if (Subtarget->isTarget64BitILP32() && !isUInt<31>(Val) &&
1767 !AM.hasBaseOrIndexReg())
1774bool X86DAGToDAGISel::matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
1775 bool AllowSegmentRegForX32) {
1787 if (
isNullConstant(Address) && AM.Segment.getNode() ==
nullptr &&
1788 !IndirectTlsSegRefs &&
1789 (Subtarget->isTargetGlibc() || Subtarget->isTargetAndroid() ||
1790 Subtarget->isTargetFuchsia())) {
1791 if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32)
1793 switch (
N->getPointerInfo().getAddrSpace()) {
1795 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
1798 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
1811bool X86DAGToDAGISel::matchWrapper(
SDValue N, X86ISelAddressMode &AM) {
1814 if (AM.hasSymbolicDisplacement())
1817 bool IsRIPRelTLS =
false;
1831 if (Subtarget->is64Bit() &&
1837 if (IsRIPRel && AM.hasBaseOrIndexReg())
1841 X86ISelAddressMode Backup = AM;
1845 if (
auto *
G = dyn_cast<GlobalAddressSDNode>(N0)) {
1846 AM.GV =
G->getGlobal();
1847 AM.SymbolFlags =
G->getTargetFlags();
1849 }
else if (
auto *CP = dyn_cast<ConstantPoolSDNode>(N0)) {
1850 AM.CP =
CP->getConstVal();
1851 AM.Alignment =
CP->getAlign();
1852 AM.SymbolFlags =
CP->getTargetFlags();
1854 }
else if (
auto *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
1855 AM.ES = S->getSymbol();
1856 AM.SymbolFlags = S->getTargetFlags();
1857 }
else if (
auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
1858 AM.MCSym = S->getMCSymbol();
1859 }
else if (
auto *J = dyn_cast<JumpTableSDNode>(N0)) {
1860 AM.JT = J->getIndex();
1861 AM.SymbolFlags = J->getTargetFlags();
1862 }
else if (
auto *BA = dyn_cast<BlockAddressSDNode>(N0)) {
1863 AM.BlockAddr = BA->getBlockAddress();
1864 AM.SymbolFlags = BA->getTargetFlags();
1865 Offset = BA->getOffset();
1869 if (foldOffsetIntoAddress(
Offset, AM)) {
1875 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64));
1883bool X86DAGToDAGISel::matchAddress(
SDValue N, X86ISelAddressMode &AM) {
1884 if (matchAddressRecursively(
N, AM, 0))
1891 if (Subtarget->isTarget64BitILP32() &&
1892 AM.BaseType == X86ISelAddressMode::RegBase &&
1893 AM.Base_Reg.getNode() !=
nullptr && AM.IndexReg.getNode() ==
nullptr) {
1894 SDValue Save_Base_Reg = AM.Base_Reg;
1895 if (
auto *LoadN = dyn_cast<LoadSDNode>(Save_Base_Reg)) {
1897 if (matchLoadInAddress(LoadN, AM,
true))
1898 AM.Base_Reg = Save_Base_Reg;
1904 if (AM.Scale == 2 &&
1905 AM.BaseType == X86ISelAddressMode::RegBase &&
1906 AM.Base_Reg.getNode() ==
nullptr) {
1907 AM.Base_Reg = AM.IndexReg;
1914 switch (
TM.getCodeModel()) {
1918 if (Subtarget->is64Bit() &&
1920 AM.BaseType == X86ISelAddressMode::RegBase &&
1921 AM.Base_Reg.getNode() ==
nullptr &&
1922 AM.IndexReg.getNode() ==
nullptr &&
1924 AM.hasSymbolicDisplacement())
1925 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64);
1932bool X86DAGToDAGISel::matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
1938 X86ISelAddressMode Backup = AM;
1939 if (!matchAddressRecursively(
N.getOperand(0), AM,
Depth+1) &&
1940 !matchAddressRecursively(Handle.getValue().getOperand(1), AM,
Depth+1))
1945 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM,
1947 !matchAddressRecursively(Handle.getValue().getOperand(0), AM,
Depth + 1))
1954 if (AM.BaseType == X86ISelAddressMode::RegBase &&
1955 !AM.Base_Reg.getNode() &&
1956 !AM.IndexReg.getNode()) {
1957 N = Handle.getValue();
1958 AM.Base_Reg =
N.getOperand(0);
1959 AM.IndexReg =
N.getOperand(1);
1963 N = Handle.getValue();
1973 if (
N->getNodeId() == -1 ||
1993 X86ISelAddressMode &AM) {
2000 if (ScaleLog <= 0 || ScaleLog >= 4 ||
2001 Mask != (0xffu << ScaleLog))
2004 MVT XVT =
X.getSimpleValueType();
2005 MVT VT =
N.getSimpleValueType();
2030 AM.Scale = (1 << ScaleLog);
2038 X86ISelAddressMode &AM) {
2044 int64_t Mask = cast<ConstantSDNode>(
N->getOperand(1))->getSExtValue();
2049 bool FoundAnyExtend =
false;
2053 FoundAnyExtend =
true;
2071 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3)
2074 MVT VT =
N.getSimpleValueType();
2076 if (FoundAnyExtend) {
2097 AM.Scale = 1 << ShiftAmt;
2098 AM.IndexReg = NewAnd;
2132 X86ISelAddressMode &AM) {
2138 unsigned MaskIdx, MaskLen;
2141 unsigned MaskLZ = 64 - (MaskIdx + MaskLen);
2147 unsigned AMShiftAmt = MaskIdx;
2151 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2155 unsigned ScaleDown = (64 -
X.getSimpleValueType().getSizeInBits()) + ShiftAmt;
2156 if (MaskLZ < ScaleDown)
2158 MaskLZ -= ScaleDown;
2166 bool ReplacingAnyExtend =
false;
2168 unsigned ExtendBits =
X.getSimpleValueType().getSizeInBits() -
2169 X.getOperand(0).getSimpleValueType().getSizeInBits();
2172 X =
X.getOperand(0);
2173 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits;
2174 ReplacingAnyExtend =
true;
2176 APInt MaskedHighBits =
2183 MVT VT =
N.getSimpleValueType();
2184 if (ReplacingAnyExtend) {
2185 assert(
X.getValueType() != VT);
2192 MVT XVT =
X.getSimpleValueType();
2213 AM.Scale = 1 << AMShiftAmt;
2214 AM.IndexReg = NewExt;
2224 X86ISelAddressMode &AM,
2232 if (!Subtarget.hasTBM() &&
2233 !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR()))
2237 unsigned MaskIdx, MaskLen;
2245 unsigned AMShiftAmt = MaskIdx;
2249 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2251 MVT XVT =
X.getSimpleValueType();
2252 MVT VT =
N.getSimpleValueType();
2277 AM.Scale = 1 << AMShiftAmt;
2278 AM.IndexReg = NewExt;
2285 X86ISelAddressMode &AM,
2287 assert(AM.IndexReg.getNode() ==
nullptr &&
"IndexReg already matched");
2288 assert((AM.Scale == 1 || AM.Scale == 2 || AM.Scale == 4 || AM.Scale == 8) &&
2289 "Illegal index scale");
2295 EVT VT =
N.getValueType();
2296 unsigned Opc =
N.getOpcode();
2299 if (CurDAG->isBaseWithConstantOffset(
N)) {
2300 auto *AddVal = cast<ConstantSDNode>(
N.getOperand(1));
2302 if (!foldOffsetIntoAddress(
Offset, AM))
2303 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2307 if (Opc ==
ISD::ADD &&
N.getOperand(0) ==
N.getOperand(1)) {
2308 if (AM.Scale <= 4) {
2310 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2316 uint64_t ShiftAmt =
N.getConstantOperandVal(1);
2317 uint64_t ScaleAmt = 1ULL << ShiftAmt;
2318 if ((AM.Scale * ScaleAmt) <= 8) {
2319 AM.Scale *= ScaleAmt;
2320 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2328 if (Src.getOpcode() ==
ISD::ADD && Src->getFlags().hasNoSignedWrap() &&
2330 if (CurDAG->isBaseWithConstantOffset(Src)) {
2331 SDValue AddSrc = Src.getOperand(0);
2332 auto *AddVal = cast<ConstantSDNode>(Src.getOperand(1));
2334 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2336 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2342 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2343 CurDAG->RemoveDeadNode(
N.getNode());
2355 unsigned SrcOpc = Src.getOpcode();
2356 if (((SrcOpc ==
ISD::ADD && Src->getFlags().hasNoUnsignedWrap()) ||
2357 CurDAG->isADDLike(Src)) &&
2359 if (CurDAG->isBaseWithConstantOffset(Src)) {
2360 SDValue AddSrc = Src.getOperand(0);
2361 auto *AddVal = cast<ConstantSDNode>(Src.getOperand(1));
2363 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2374 if ((AM.Scale * ScaleAmt) <= 8 &&
2376 CurDAG->MaskedValueIsZero(ShVal, HiBits))) {
2377 AM.Scale *= ScaleAmt;
2378 SDValue ExtShVal = CurDAG->getNode(Opc,
DL, VT, ShVal);
2387 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2389 SDValue ExtAdd = CurDAG->getNode(SrcOpc,
DL, VT, ExtSrc, ExtVal);
2393 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2394 CurDAG->RemoveDeadNode(
N.getNode());
2395 return Res ? Res : ExtSrc;
2405bool X86DAGToDAGISel::matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
2409 dbgs() <<
"MatchAddress: ";
2414 return matchAddressBase(
N, AM);
2419 if (AM.isRIPRelative()) {
2423 if (!(AM.ES || AM.MCSym) && AM.JT != -1)
2426 if (
auto *Cst = dyn_cast<ConstantSDNode>(
N))
2427 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM))
2432 switch (
N.getOpcode()) {
2435 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
2436 if (
const auto *ESNode = dyn_cast<MCSymbolSDNode>(
N.getOperand(0))) {
2438 AM.MCSym = ESNode->getMCSymbol();
2444 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2445 if (!foldOffsetIntoAddress(Val, AM))
2452 if (!matchWrapper(
N, AM))
2457 if (!matchLoadInAddress(cast<LoadSDNode>(
N), AM))
2462 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2463 AM.Base_Reg.getNode() ==
nullptr &&
2465 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
2466 AM.Base_FrameIndex = cast<FrameIndexSDNode>(
N)->getIndex();
2472 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2475 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
2476 unsigned Val = CN->getZExtValue();
2481 if (Val == 1 || Val == 2 || Val == 3) {
2483 AM.Scale = 1 << Val;
2484 AM.IndexReg = matchIndexRecursively(ShVal, AM,
Depth + 1);
2492 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2496 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2497 "Unexpected value size!");
2506 if (!isa<ConstantSDNode>(
N.getOperand(1)) ||
2507 !isa<ConstantSDNode>(
And.getOperand(1)))
2509 uint64_t Mask =
And.getConstantOperandVal(1) >>
N.getConstantOperandVal(1);
2521 if (
N.getResNo() != 0)
break;
2526 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2527 AM.Base_Reg.getNode() ==
nullptr &&
2528 AM.IndexReg.getNode() ==
nullptr) {
2529 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
2530 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
2531 CN->getZExtValue() == 9) {
2532 AM.Scale =
unsigned(CN->getZExtValue())-1;
2543 auto *AddVal = cast<ConstantSDNode>(MulVal.
getOperand(1));
2544 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue();
2545 if (foldOffsetIntoAddress(Disp, AM))
2546 Reg =
N.getOperand(0);
2548 Reg =
N.getOperand(0);
2551 AM.IndexReg = AM.Base_Reg =
Reg;
2570 X86ISelAddressMode Backup = AM;
2571 if (matchAddressRecursively(
N.getOperand(0), AM,
Depth+1)) {
2572 N = Handle.getValue();
2576 N = Handle.getValue();
2578 if (AM.IndexReg.getNode() || AM.isRIPRelative()) {
2593 RHS.getOperand(0).getValueType() == MVT::i32))
2597 if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() &&
2598 !AM.Base_Reg.getNode()->hasOneUse()) ||
2599 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
2603 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
2604 ((AM.Disp != 0) && (Backup.Disp == 0)) +
2605 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
2617 AM.NegateIndex =
true;
2625 if (!CurDAG->isADDLike(
N))
2629 if (!matchAdd(
N, AM,
Depth))
2638 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2642 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2643 "Unexpected value size!");
2645 if (!isa<ConstantSDNode>(
N.getOperand(1)))
2648 if (
N.getOperand(0).getOpcode() ==
ISD::SRL) {
2677 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2687 AM.IndexReg =
Index;
2693 if (Src.getOpcode() ==
ISD::AND && Src.hasOneUse())
2694 if (
auto *MaskC = dyn_cast<ConstantSDNode>(Src.getOperand(1))) {
2695 Mask = MaskC->getAPIntValue();
2696 Src = Src.getOperand(0);
2699 if (Src.getOpcode() ==
ISD::SHL && Src.hasOneUse()) {
2701 SDValue ShlSrc = Src.getOperand(0);
2702 SDValue ShlAmt = Src.getOperand(1);
2703 auto *ShAmtC = dyn_cast<ConstantSDNode>(ShlAmt);
2706 unsigned ShAmtV = ShAmtC->getZExtValue();
2714 if (!Src->getFlags().hasNoUnsignedWrap() &&
2715 !CurDAG->MaskedValueIsZero(ShlSrc, HighZeros & Mask))
2723 MVT VT =
N.getSimpleValueType();
2727 if (!
Mask.isAllOnes()) {
2728 Res = CurDAG->getConstant(
Mask.lshr(ShAmtV),
DL, SrcVT);
2730 Res = CurDAG->getNode(
ISD::AND,
DL, SrcVT, ShlSrc, Res);
2739 AM.Scale = 1 << ShAmtV;
2742 CurDAG->ReplaceAllUsesWith(
N, NewShl);
2743 CurDAG->RemoveDeadNode(
N.getNode());
2747 if (Src.getOpcode() ==
ISD::SRL && !
Mask.isAllOnes()) {
2750 Src.getOperand(0), AM))
2755 Src.getOperand(0), AM))
2760 Src.getOperand(0), AM, *Subtarget))
2768 return matchAddressBase(
N, AM);
2773bool X86DAGToDAGISel::matchAddressBase(
SDValue N, X86ISelAddressMode &AM) {
2775 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
2777 if (!AM.IndexReg.getNode()) {
2788 AM.BaseType = X86ISelAddressMode::RegBase;
2793bool X86DAGToDAGISel::matchVectorAddressRecursively(
SDValue N,
2794 X86ISelAddressMode &AM,
2798 dbgs() <<
"MatchVectorAddress: ";
2803 return matchAddressBase(
N, AM);
2806 switch (
N.getOpcode()) {
2808 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2809 if (!foldOffsetIntoAddress(Val, AM))
2814 if (!matchWrapper(
N, AM))
2822 X86ISelAddressMode Backup = AM;
2823 if (!matchVectorAddressRecursively(
N.getOperand(0), AM,
Depth + 1) &&
2824 !matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2830 if (!matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2832 !matchVectorAddressRecursively(Handle.getValue().getOperand(0), AM,
2837 N = Handle.getValue();
2842 return matchAddressBase(
N, AM);
2848bool X86DAGToDAGISel::matchVectorAddress(
SDValue N, X86ISelAddressMode &AM) {
2849 return matchVectorAddressRecursively(
N, AM, 0);
2857 X86ISelAddressMode AM;
2858 AM.Scale = cast<ConstantSDNode>(ScaleOp)->getZExtValue();
2863 AM.IndexReg = matchIndexRecursively(IndexOp, AM, 0);
2865 AM.IndexReg = IndexOp;
2869 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2871 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2873 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2879 if (matchVectorAddress(BasePtr, AM))
2882 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2896 X86ISelAddressMode AM;
2908 unsigned AddrSpace =
2909 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
2911 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2913 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2915 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2920 MVT VT =
N.getSimpleValueType();
2922 if (matchAddress(
N, AM))
2925 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2935 N =
N.getOperand(0);
2946 std::optional<ConstantRange> CR =
2947 cast<GlobalAddressSDNode>(
N)->getGlobal()->getAbsoluteSymbolRange();
2951 return CR->getUnsignedMax().ult(1ull << 32);
2960 if (!selectLEAAddr(
N,
Base, Scale,
Index, Disp, Segment))
2963 auto *
RN = dyn_cast<RegisterSDNode>(
Base);
2964 if (RN &&
RN->getReg() == 0)
2965 Base = CurDAG->getRegister(0, MVT::i64);
2966 else if (
Base.getValueType() == MVT::i32 && !isa<FrameIndexSDNode>(
Base)) {
2970 Base = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
2974 RN = dyn_cast<RegisterSDNode>(
Index);
2975 if (RN &&
RN->getReg() == 0)
2976 Index = CurDAG->getRegister(0, MVT::i64);
2979 "Expect to be extending 32-bit registers for use in LEA");
2982 Index = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
2991bool X86DAGToDAGISel::selectLEAAddr(
SDValue N,
2995 X86ISelAddressMode AM;
2999 MVT VT =
N.getSimpleValueType();
3004 SDValue T = CurDAG->getRegister(0, MVT::i32);
3006 if (matchAddress(
N, AM))
3011 unsigned Complexity = 0;
3012 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode())
3014 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
3017 if (AM.IndexReg.getNode())
3030 if (AM.hasSymbolicDisplacement()) {
3032 if (Subtarget->is64Bit())
3042 auto isMathWithFlags = [](
SDValue V) {
3043 switch (
V.getOpcode()) {
3064 if (isMathWithFlags(
N.getOperand(0)) || isMathWithFlags(
N.getOperand(1)))
3072 if (Complexity <= 2)
3075 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
3084 auto *GA = cast<GlobalAddressSDNode>(
N);
3086 X86ISelAddressMode AM;
3087 AM.GV = GA->getGlobal();
3088 AM.Disp += GA->getOffset();
3089 AM.SymbolFlags = GA->getTargetFlags();
3091 if (Subtarget->is32Bit()) {
3093 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
3096 MVT VT =
N.getSimpleValueType();
3097 getAddressOperands(AM,
SDLoc(
N), VT,
Base, Scale,
Index, Disp, Segment);
3105 EVT VT =
N.getValueType();
3106 bool WasTruncated =
false;
3108 WasTruncated =
true;
3109 N =
N.getOperand(0);
3118 unsigned Opc =
N.getOperand(0)->getOpcode();
3120 Op =
N.getOperand(0);
3123 return !WasTruncated;
3127 auto *GA = cast<GlobalAddressSDNode>(
N.getOperand(0));
3128 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange();
3129 if (!CR || CR->getUnsignedMax().uge(1ull << VT.
getSizeInBits()))
3133 Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(),
SDLoc(
N), VT,
3134 GA->getOffset(), GA->getTargetFlags());
3142 assert(Root &&
P &&
"Unknown root/parent nodes");
3144 !IsProfitableToFold(
N,
P, Root) ||
3145 !IsLegalToFold(
N,
P, Root, OptLevel))
3148 return selectAddr(
N.getNode(),
3149 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3156 assert(Root &&
P &&
"Unknown root/parent nodes");
3158 !IsProfitableToFold(
N,
P, Root) ||
3159 !IsLegalToFold(
N,
P, Root, OptLevel))
3162 return selectAddr(
N.getNode(),
3163 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3169SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
3170 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
3172 return CurDAG->getRegister(GlobalBaseReg, TLI->
getPointerTy(
DL)).getNode();
3175bool X86DAGToDAGISel::isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const {
3177 N =
N->getOperand(0).getNode();
3181 auto *GA = dyn_cast<GlobalAddressSDNode>(
N->getOperand(0));
3185 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange();
3189 return CR->getSignedMin().sge(-1ull << Width) &&
3190 CR->getSignedMax().slt(1ull << Width);
3194 assert(
N->isMachineOpcode() &&
"Unexpected node");
3195 unsigned Opc =
N->getMachineOpcode();
3196 const MCInstrDesc &MCID = getInstrInfo()->get(Opc);
3201 return static_cast<X86::CondCode>(
N->getConstantOperandVal(CondNo));
3206bool X86DAGToDAGISel::onlyUsesZeroFlag(
SDValue Flags)
const {
3211 if (UI.getUse().getResNo() !=
Flags.getResNo())
3215 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3219 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3221 if (FlagUI.getUse().getResNo() != 1)
continue;
3223 if (!FlagUI->isMachineOpcode())
return false;
3242bool X86DAGToDAGISel::hasNoSignFlagUses(
SDValue Flags)
const {
3247 if (UI.getUse().getResNo() !=
Flags.getResNo())
3251 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3255 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3257 if (FlagUI.getUse().getResNo() != 1)
continue;
3259 if (!FlagUI->isMachineOpcode())
return false;
3298 bool X86DAGToDAGISel::hasNoCarryFlagUses(
SDValue Flags)
const {
3303 if (UI.getUse().getResNo() !=
Flags.getResNo())
3306 unsigned UIOpc = UI->getOpcode();
3310 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3314 FlagUI != FlagUE; ++FlagUI) {
3316 if (FlagUI.getUse().getResNo() != 1)
3319 if (!FlagUI->isMachineOpcode())
3360 if (StoredVal.
getResNo() != 0)
return false;
3374 LoadNode = cast<LoadSDNode>(Load);
3377 if (!Load.hasOneUse())
3385 bool FoundLoad =
false;
3389 const unsigned int Max = 1024;
3431 if (Chain == Load.getValue(1)) {
3437 if (
Op == Load.getValue(1)) {
3453 if (
Op.getNode() != LoadNode)
3486bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(
SDNode *
Node) {
3487 auto *StoreNode = cast<StoreSDNode>(
Node);
3494 EVT MemVT = StoreNode->getMemoryVT();
3495 if (MemVT != MVT::i64 && MemVT != MVT::i32 && MemVT != MVT::i16 &&
3499 bool IsCommutable =
false;
3500 bool IsNegate =
false;
3514 IsCommutable =
true;
3518 unsigned LoadOpNo = IsNegate ? 1 : 0;
3522 LoadNode, InputChain)) {
3529 LoadNode, InputChain))
3538 auto SelectOpcode = [&](
unsigned Opc64,
unsigned Opc32,
unsigned Opc16,
3559 unsigned NewOpc = SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m,
3569 if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) {
3573 if ((IsOne || IsNegOne) && hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3576 ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
3577 : SelectOpcode(
X86::DEC64m,
X86::DEC32m,
X86::DEC16m,
X86::DEC8m);
3590 auto SelectRegOpcode = [SelectOpcode](
unsigned Opc) {
3593 return SelectOpcode(X86::ADD64mr, X86::ADD32mr, X86::ADD16mr,
3596 return SelectOpcode(X86::ADC64mr, X86::ADC32mr, X86::ADC16mr,
3599 return SelectOpcode(X86::SUB64mr, X86::SUB32mr, X86::SUB16mr,
3602 return SelectOpcode(X86::SBB64mr, X86::SBB32mr, X86::SBB16mr,
3605 return SelectOpcode(X86::AND64mr, X86::AND32mr, X86::AND16mr,
3608 return SelectOpcode(X86::OR64mr, X86::OR32mr, X86::OR16mr, X86::OR8mr);
3610 return SelectOpcode(X86::XOR64mr, X86::XOR32mr, X86::XOR16mr,
3616 auto SelectImmOpcode = [SelectOpcode](
unsigned Opc) {
3619 return SelectOpcode(X86::ADD64mi32, X86::ADD32mi, X86::ADD16mi,
3622 return SelectOpcode(X86::ADC64mi32, X86::ADC32mi, X86::ADC16mi,
3625 return SelectOpcode(X86::SUB64mi32, X86::SUB32mi, X86::SUB16mi,
3628 return SelectOpcode(X86::SBB64mi32, X86::SBB32mi, X86::SBB16mi,
3631 return SelectOpcode(X86::AND64mi32, X86::AND32mi, X86::AND16mi,
3634 return SelectOpcode(X86::OR64mi32, X86::OR32mi, X86::OR16mi,
3637 return SelectOpcode(X86::XOR64mi32, X86::XOR32mi, X86::XOR16mi,
3644 unsigned NewOpc = SelectRegOpcode(Opc);
3649 if (
auto *OperandC = dyn_cast<ConstantSDNode>(Operand)) {
3650 int64_t OperandV = OperandC->getSExtValue();
3656 ((MemVT != MVT::i8 && !isInt<8>(OperandV) && isInt<8>(-OperandV)) ||
3657 (MemVT == MVT::i64 && !isInt<32>(OperandV) &&
3658 isInt<32>(-OperandV))) &&
3659 hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3660 OperandV = -OperandV;
3664 if (MemVT != MVT::i64 || isInt<32>(OperandV)) {
3665 Operand = CurDAG->getTargetConstant(OperandV,
SDLoc(
Node), MemVT);
3666 NewOpc = SelectImmOpcode(Opc);
3672 CurDAG->getCopyToReg(InputChain,
SDLoc(
Node), X86::EFLAGS,
3676 Segment, Operand, CopyTo, CopyTo.
getValue(1)};
3677 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3681 Segment, Operand, InputChain};
3682 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3693 CurDAG->setNodeMemRefs(Result, MemOps);
3699 CurDAG->RemoveDeadNode(
Node);
3710bool X86DAGToDAGISel::matchBitExtract(
SDNode *
Node) {
3714 "Should be either an and-mask, or right-shift after clearing high bits.");
3717 if (!Subtarget->hasBMI() && !Subtarget->hasBMI2())
3720 MVT NVT =
Node->getSimpleValueType(0);
3723 if (NVT != MVT::i32 && NVT != MVT::i64)
3731 const bool AllowExtraUsesByDefault = Subtarget->hasBMI2();
3732 auto checkUses = [AllowExtraUsesByDefault](
3734 std::optional<bool> AllowExtraUses) {
3735 return AllowExtraUses.value_or(AllowExtraUsesByDefault) ||
3736 Op.getNode()->hasNUsesOfValue(NUses,
Op.getResNo());
3738 auto checkOneUse = [checkUses](
SDValue Op,
3739 std::optional<bool> AllowExtraUses =
3741 return checkUses(
Op, 1, AllowExtraUses);
3743 auto checkTwoUse = [checkUses](
SDValue Op,
3744 std::optional<bool> AllowExtraUses =
3746 return checkUses(
Op, 2, AllowExtraUses);
3749 auto peekThroughOneUseTruncation = [checkOneUse](
SDValue V) {
3751 assert(
V.getSimpleValueType() == MVT::i32 &&
3752 V.getOperand(0).getSimpleValueType() == MVT::i64 &&
3753 "Expected i64 -> i32 truncation");
3754 V =
V.getOperand(0);
3760 auto matchPatternA = [checkOneUse, peekThroughOneUseTruncation, &NBits,
3763 if (
Mask->getOpcode() !=
ISD::ADD || !checkOneUse(Mask))
3769 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3774 NBits =
M0->getOperand(1);
3775 NegateNBits =
false;
3779 auto isAllOnes = [
this, peekThroughOneUseTruncation, NVT](
SDValue V) {
3780 V = peekThroughOneUseTruncation(V);
3781 return CurDAG->MaskedValueIsAllOnes(
3787 auto matchPatternB = [checkOneUse, isAllOnes, peekThroughOneUseTruncation,
3790 if (
Mask.getOpcode() !=
ISD::XOR || !checkOneUse(Mask))
3793 if (!isAllOnes(
Mask->getOperand(1)))
3796 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3800 if (!isAllOnes(
M0->getOperand(0)))
3802 NBits =
M0->getOperand(1);
3803 NegateNBits =
false;
3809 auto canonicalizeShiftAmt = [&NBits, &NegateNBits](
SDValue ShiftAmt,
3810 unsigned Bitwidth) {
3815 NBits = NBits.getOperand(0);
3820 auto *V0 = dyn_cast<ConstantSDNode>(NBits.getOperand(0));
3821 if (!V0 || V0->getZExtValue() != Bitwidth)
3823 NBits = NBits.getOperand(1);
3824 NegateNBits =
false;
3830 auto matchPatternC = [checkOneUse, peekThroughOneUseTruncation, &NegateNBits,
3833 Mask = peekThroughOneUseTruncation(Mask);
3834 unsigned Bitwidth =
Mask.getSimpleValueType().getSizeInBits();
3836 if (
Mask.getOpcode() !=
ISD::SRL || !checkOneUse(Mask))
3843 if (!checkOneUse(
M1))
3845 canonicalizeShiftAmt(
M1, Bitwidth);
3850 return !NegateNBits;
3858 auto matchPatternD = [checkOneUse, checkTwoUse, canonicalizeShiftAmt,
3859 AllowExtraUsesByDefault, &NegateNBits,
3872 canonicalizeShiftAmt(N1, Bitwidth);
3876 const bool AllowExtraUses = AllowExtraUsesByDefault && !NegateNBits;
3877 if (!checkOneUse(N0, AllowExtraUses) || !checkTwoUse(N1, AllowExtraUses))
3883 auto matchLowBitMask = [matchPatternA, matchPatternB,
3885 return matchPatternA(Mask) || matchPatternB(Mask) || matchPatternC(Mask);
3889 X =
Node->getOperand(0);
3892 if (matchLowBitMask(Mask)) {
3896 if (!matchLowBitMask(Mask))
3900 X = CurDAG->getAllOnesConstant(
SDLoc(
Node), NVT);
3901 }
else if (!matchPatternD(
Node))
3906 if (NegateNBits && !Subtarget->hasBMI2())
3918 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, MVT::i32), 0);
3921 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit,
DL, MVT::i32);
3923 NBits =
SDValue(CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG,
DL,
3924 MVT::i32, ImplDef, NBits, SRIdxVal),
3934 NBits = CurDAG->getNode(
ISD::SUB,
DL, MVT::i32, BitWidthC, NBits);
3938 if (Subtarget->hasBMI2()) {
3940 if (NVT != MVT::i32) {
3948 SelectCode(Extract.
getNode());
3957 SDValue RealX = peekThroughOneUseTruncation(
X);
3963 MVT XVT =
X.getSimpleValueType();
3973 SDValue C8 = CurDAG->getConstant(8,
DL, MVT::i8);
3981 SDValue ShiftAmt =
X.getOperand(1);
3982 X =
X.getOperand(0);
3985 "Expected shift amount to be i8");
3989 SDValue OrigShiftAmt = ShiftAmt;
3994 Control = CurDAG->getNode(
ISD::OR,
DL, MVT::i32, Control, ShiftAmt);
3999 if (XVT != MVT::i32) {
4014 SelectCode(Extract.
getNode());
4021 MVT NVT =
Node->getSimpleValueType(0);
4034 Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR());
4035 if (!PreferBEXTR && !Subtarget->hasBMI2())
4047 if (NVT != MVT::i32 && NVT != MVT::i64)
4051 auto *MaskCst = dyn_cast<ConstantSDNode>(N1);
4052 auto *ShiftCst = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
4053 if (!MaskCst || !ShiftCst)
4061 uint64_t Shift = ShiftCst->getZExtValue();
4066 if (Shift == 8 && MaskSize == 8)
4077 if (!PreferBEXTR && MaskSize <= 32)
4081 unsigned ROpc, MOpc;
4084 assert(Subtarget->hasBMI2() &&
"We must have BMI2's BZHI then.");
4088 Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT);
4089 ROpc = NVT == MVT::i64 ? X86::BZHI64rr : X86::BZHI32rr;
4090 MOpc = NVT == MVT::i64 ? X86::BZHI64rm : X86::BZHI32rm;
4091 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4092 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4098 Control = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT);
4099 if (Subtarget->hasTBM()) {
4100 ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri;
4101 MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi;
4103 assert(Subtarget->hasBMI() &&
"We must have BMI1's BEXTR then.");
4105 ROpc = NVT == MVT::i64 ? X86::BEXTR64rr : X86::BEXTR32rr;
4106 MOpc = NVT == MVT::i64 ? X86::BEXTR64rm : X86::BEXTR32rm;
4107 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4108 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4114 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4115 if (tryFoldLoad(
Node, N0.
getNode(), Input, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4117 Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Control, Input.
getOperand(0)};
4118 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
4119 NewNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4123 CurDAG->setNodeMemRefs(NewNode, {cast<LoadSDNode>(Input)->getMemOperand()});
4125 NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, Control);
4130 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT);
4131 unsigned NewOpc = NVT == MVT::i64 ? X86::SHR64ri : X86::SHR32ri;
4133 CurDAG->getMachineNode(NewOpc, dl, NVT,
SDValue(NewNode, 0), ShAmt);
4140MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
4141 bool MayFoldLoad,
const SDLoc &dl,
4146 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4147 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4150 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4151 if (MayFoldLoad && tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4152 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4154 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other);
4155 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4159 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
4164 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32);
4165 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4172MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
4173 bool MayFoldLoad,
const SDLoc &dl,
4179 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4180 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4183 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4184 if (MayFoldLoad && tryFoldLoad(
Node, N2, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4185 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4187 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other, MVT::Glue);
4188 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4193 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N2)->getMemOperand()});
4198 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Glue);
4199 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4204bool X86DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
4205 EVT VT =
N->getValueType(0);
4212 unsigned Size = VT == MVT::i64 ? 64 : 32;
4214 SDValue OrigShiftAmt =
N->getOperand(1);
4215 SDValue ShiftAmt = OrigShiftAmt;
4230 auto *Add0C = dyn_cast<ConstantSDNode>(Add0);
4231 auto *Add1C = dyn_cast<ConstantSDNode>(Add1);
4234 if (Add1C && Add1C->getAPIntValue().urem(
Size) == 0) {
4238 ((Add0C && Add0C->getAPIntValue().urem(
Size) ==
Size - 1) ||
4239 (Add1C && Add1C->getAPIntValue().urem(
Size) ==
Size - 1))) {
4243 assert(Add0C ==
nullptr || Add1C ==
nullptr);
4252 NewShiftAmt = CurDAG->getNode(
ISD::XOR,
DL, OpVT,
4253 Add0C ==
nullptr ? Add0 : Add1,
AllOnes);
4259 Add0C->getZExtValue() != 0) {
4262 if (Add0C->getZExtValue() %
Size == 0)
4265 Add0C->getZExtValue() % 32 == 0) {
4273 Add0 = CurDAG->getZExtOrTrunc(Add0,
DL, SubVT);
4277 X = CurDAG->getNode(
ISD::ADD,
DL, SubVT, Add1, Add0);
4299 NewShiftAmt = CurDAG->getNode(
ISD::TRUNCATE,
DL, MVT::i8, NewShiftAmt);
4306 NewShiftAmt = CurDAG->getNode(
ISD::AND,
DL, MVT::i8, NewShiftAmt,
4307 CurDAG->getConstant(
Size - 1,
DL, MVT::i8));
4311 SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(
N,
N->getOperand(0),
4313 if (UpdatedNode !=
N) {
4316 ReplaceNode(
N, UpdatedNode);
4323 CurDAG->RemoveDeadNode(OrigShiftAmt.
getNode());
4331bool X86DAGToDAGISel::tryShrinkShlLogicImm(
SDNode *
N) {
4332 MVT NVT =
N->getSimpleValueType(0);
4333 unsigned Opcode =
N->getOpcode();
4341 auto *Cst = dyn_cast<ConstantSDNode>(N1);
4345 int64_t Val = Cst->getSExtValue();
4350 bool FoundAnyExtend =
false;
4354 FoundAnyExtend =
true;
4362 if (NVT != MVT::i32 && NVT != MVT::i64)
4365 auto *ShlCst = dyn_cast<ConstantSDNode>(Shift.
getOperand(1));
4369 uint64_t ShAmt = ShlCst->getZExtValue();
4373 uint64_t RemovedBitsMask = (1ULL << ShAmt) - 1;
4379 auto CanShrinkImmediate = [&](int64_t &ShiftedVal) {
4383 ShiftedVal = (
uint64_t)Val >> ShAmt;
4384 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4387 if (ShiftedVal == UINT8_MAX || ShiftedVal == UINT16_MAX)
4390 ShiftedVal = Val >> ShAmt;
4391 if ((!isInt<8>(Val) && isInt<8>(ShiftedVal)) ||
4392 (!isInt<32>(Val) && isInt<32>(ShiftedVal)))
4396 ShiftedVal = (
uint64_t)Val >> ShAmt;
4397 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4404 if (!CanShrinkImmediate(ShiftedVal))
4414 unsigned ZExtWidth = Cst->getAPIntValue().getActiveBits();
4420 NeededMask &= ~Cst->getAPIntValue();
4422 if (CurDAG->MaskedValueIsZero(
N->getOperand(0), NeededMask))
4427 if (FoundAnyExtend) {
4433 SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT);
4444bool X86DAGToDAGISel::matchVPTERNLOG(
SDNode *Root,
SDNode *ParentA,
4448 assert(
A.isOperandOf(ParentA) &&
B.isOperandOf(ParentB) &&
4449 C.isOperandOf(ParentC) &&
"Incorrect parent node");
4451 auto tryFoldLoadOrBCast =
4454 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4460 L =
L.getOperand(0);
4467 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4468 unsigned Size = MemIntr->getMemoryVT().getSizeInBits();
4472 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4475 bool FoldedLoad =
false;
4476 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4477 if (tryFoldLoadOrBCast(Root, ParentC,
C, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4479 }
else if (tryFoldLoadOrBCast(Root, ParentA,
A, Tmp0, Tmp1, Tmp2, Tmp3,
4484 uint8_t OldImm =
Imm;
4485 Imm = OldImm & 0xa5;
4486 if (OldImm & 0x02)
Imm |= 0x10;
4487 if (OldImm & 0x10)
Imm |= 0x02;
4488 if (OldImm & 0x08)
Imm |= 0x40;
4489 if (OldImm & 0x40)
Imm |= 0x08;
4490 }
else if (tryFoldLoadOrBCast(Root, ParentB,
B, Tmp0, Tmp1, Tmp2, Tmp3,
4495 uint8_t OldImm =
Imm;
4496 Imm = OldImm & 0x99;
4497 if (OldImm & 0x02)
Imm |= 0x04;
4498 if (OldImm & 0x04)
Imm |= 0x02;
4499 if (OldImm & 0x20)
Imm |= 0x40;
4500 if (OldImm & 0x40)
Imm |= 0x20;
4505 SDValue TImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
4511 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
4515 auto *MemIntr = cast<MemIntrinsicSDNode>(
C);
4516 unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits();
4517 assert((EltSize == 32 || EltSize == 64) &&
"Unexpected broadcast size!");
4519 bool UseD = EltSize == 32;
4521 Opc = UseD ? X86::VPTERNLOGDZ128rmbi : X86::VPTERNLOGQZ128rmbi;
4523 Opc = UseD ? X86::VPTERNLOGDZ256rmbi : X86::VPTERNLOGQZ256rmbi;
4525 Opc = UseD ? X86::VPTERNLOGDZrmbi : X86::VPTERNLOGQZrmbi;
4531 Opc = UseD ? X86::VPTERNLOGDZ128rmi : X86::VPTERNLOGQZ128rmi;
4533 Opc = UseD ? X86::VPTERNLOGDZ256rmi : X86::VPTERNLOGQZ256rmi;
4535 Opc = UseD ? X86::VPTERNLOGDZrmi : X86::VPTERNLOGQZrmi;
4540 SDValue Ops[] = {
A,
B, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, TImm,
C.getOperand(0)};
4541 MNode = CurDAG->getMachineNode(Opc,
DL, VTs, Ops);
4544 ReplaceUses(
C.getValue(1),
SDValue(MNode, 1));
4546 CurDAG->setNodeMemRefs(MNode, {cast<MemSDNode>(
C)->getMemOperand()});
4551 Opc = UseD ? X86::VPTERNLOGDZ128rri : X86::VPTERNLOGQZ128rri;
4553 Opc = UseD ? X86::VPTERNLOGDZ256rri : X86::VPTERNLOGQZ256rri;
4555 Opc = UseD ? X86::VPTERNLOGDZrri : X86::VPTERNLOGQZrri;
4559 MNode = CurDAG->getMachineNode(Opc,
DL, NVT, {
A,
B,
C, TImm});
4563 CurDAG->RemoveDeadNode(Root);
4569bool X86DAGToDAGISel::tryVPTERNLOG(
SDNode *
N) {
4570 MVT NVT =
N->getSimpleValueType(0);
4573 if (!NVT.
isVector() || !Subtarget->hasAVX512() ||
4584 auto getFoldableLogicOp = [](
SDValue Op) {
4587 Op =
Op.getOperand(0);
4589 if (!
Op.hasOneUse())
4592 unsigned Opc =
Op.getOpcode();
4601 if ((FoldableOp = getFoldableLogicOp(N1))) {
4603 }
else if ((FoldableOp = getFoldableLogicOp(N0))) {
4616 uint8_t TernlogMagicA = 0xf0;
4617 uint8_t TernlogMagicB = 0xcc;
4618 uint8_t TernlogMagicC = 0xaa;
4627 Parent =
Op.getNode();
4628 Op =
Op.getOperand(0);
4632 PeekThroughNot(
A, ParentA, TernlogMagicA);
4633 PeekThroughNot(
B, ParentB, TernlogMagicB);
4634 PeekThroughNot(
C, ParentC, TernlogMagicC);
4639 case ISD::AND:
Imm = TernlogMagicB & TernlogMagicC;
break;
4640 case ISD::OR:
Imm = TernlogMagicB | TernlogMagicC;
break;
4641 case ISD::XOR:
Imm = TernlogMagicB ^ TernlogMagicC;
break;
4645 switch (
N->getOpcode()) {
4649 Imm &= ~TernlogMagicA;
4651 Imm = ~(
Imm) & TernlogMagicA;
4658 return matchVPTERNLOG(
N, ParentA, ParentB, ParentC,
A,
B,
C, Imm);
4668bool X86DAGToDAGISel::shrinkAndImmediate(
SDNode *
And) {
4671 MVT VT =
And->getSimpleValueType(0);
4672 if (VT != MVT::i32 && VT != MVT::i64)
4675 auto *And1C = dyn_cast<ConstantSDNode>(
And->getOperand(1));
4684 APInt MaskVal = And1C->getAPIntValue();
4686 if (!MaskLZ || (VT == MVT::i64 && MaskLZ == 32))
4690 if (VT == MVT::i64 && MaskLZ >= 32) {
4692 MaskVal = MaskVal.
trunc(32);
4697 APInt NegMaskVal = MaskVal | HighZeros;
4706 if (VT == MVT::i64 && MaskVal.
getBitWidth() < 64) {
4707 NegMaskVal = NegMaskVal.
zext(64);
4708 HighZeros = HighZeros.
zext(64);
4713 if (!CurDAG->MaskedValueIsZero(And0, HighZeros))
4733 bool FoldedBCast,
bool Masked) {
4734#define VPTESTM_CASE(VT, SUFFIX) \
4737 return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
4738 return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
4741#define VPTESTM_BROADCAST_CASES(SUFFIX) \
4742default: llvm_unreachable("Unexpected VT!"); \
4743VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
4744VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
4745VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
4746VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
4747VPTESTM_CASE(v16i32, DZ##SUFFIX) \
4748VPTESTM_CASE(v8i64, QZ##SUFFIX)
4750#define VPTESTM_FULL_CASES(SUFFIX) \
4751VPTESTM_BROADCAST_CASES(SUFFIX) \
4752VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
4753VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
4754VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
4755VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
4756VPTESTM_CASE(v64i8, BZ##SUFFIX) \
4757VPTESTM_CASE(v32i16, WZ##SUFFIX)
4775#undef VPTESTM_FULL_CASES
4776#undef VPTESTM_BROADCAST_CASES
4782bool X86DAGToDAGISel::tryVPTESTM(
SDNode *Root,
SDValue Setcc,
4784 assert(Subtarget->hasAVX512() &&
"Expected AVX512!");
4834 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4839 if (CmpSVT != MVT::i32 && CmpSVT != MVT::i64)
4845 L =
L.getOperand(0);
4851 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4852 if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.
getSizeInBits())
4855 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4859 bool CanFoldLoads = Src0 != Src1;
4861 bool FoldedLoad =
false;
4862 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4864 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src1, Tmp0, Tmp1, Tmp2,