22#include "llvm/Config/llvm-config.h"
27#include "llvm/IR/IntrinsicsX86.h"
37#define DEBUG_TYPE "x86-isel"
38#define PASS_NAME "X86 DAG->DAG Instruction Selection"
40STATISTIC(NumLoadMoved,
"Number of loads moved below TokenFactor");
43 cl::desc(
"Enable setting constant bits to reduce size of mask immediates"),
47 "x86-promote-anyext-load",
cl::init(
true),
59 struct X86ISelAddressMode {
67 int Base_FrameIndex = 0;
76 const char *ES =
nullptr;
81 bool NegateIndex =
false;
83 X86ISelAddressMode() =
default;
85 bool hasSymbolicDisplacement()
const {
86 return GV !=
nullptr ||
CP !=
nullptr || ES !=
nullptr ||
87 MCSym !=
nullptr ||
JT != -1 || BlockAddr !=
nullptr;
90 bool hasBaseOrIndexReg()
const {
97 if (
BaseType != RegBase)
return false;
99 dyn_cast_or_null<RegisterSDNode>(Base_Reg.
getNode()))
100 return RegNode->getReg() == X86::RIP;
109#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
111 dbgs() <<
"X86ISelAddressMode " <<
this <<
'\n';
112 dbgs() <<
"Base_Reg ";
118 dbgs() <<
" Base.FrameIndex " << Base_FrameIndex <<
'\n';
119 dbgs() <<
" Scale " << Scale <<
'\n'
127 dbgs() <<
" Disp " << Disp <<
'\n'
149 dbgs() <<
" JT" <<
JT <<
" Align" << Alignment.
value() <<
'\n';
169 bool IndirectTlsSegRefs;
174 X86DAGToDAGISel() =
delete;
178 OptForMinSize(
false), IndirectTlsSegRefs(
false) {}
184 "indirect-tls-seg-refs");
189 "OptForMinSize implies OptForSize");
203#include "X86GenDAGISel.inc"
208 bool foldOffsetIntoAddress(
uint64_t Offset, X86ISelAddressMode &AM);
209 bool matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
210 bool AllowSegmentRegForX32 =
false);
211 bool matchWrapper(
SDValue N, X86ISelAddressMode &AM);
212 bool matchAddress(
SDValue N, X86ISelAddressMode &AM);
213 bool matchVectorAddress(
SDValue N, X86ISelAddressMode &AM);
214 bool matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
unsigned Depth);
215 bool matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
217 bool matchVectorAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
219 bool matchAddressBase(
SDValue N, X86ISelAddressMode &AM);
248 return tryFoldLoad(
P,
P,
N,
Base, Scale,
Index, Disp, Segment);
256 bool isProfitableToFormMaskedOp(
SDNode *
N)
const;
260 unsigned ConstraintID,
261 std::vector<SDValue> &OutOps)
override;
263 void emitSpecialCodeForMain();
265 inline void getAddressOperands(X86ISelAddressMode &AM,
const SDLoc &
DL,
269 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
270 Base = CurDAG->getTargetFrameIndex(
271 AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout()));
272 else if (AM.Base_Reg.getNode())
275 Base = CurDAG->getRegister(0, VT);
277 Scale = getI8Imm(AM.Scale,
DL);
280 if (AM.NegateIndex) {
281 unsigned NegOpc = VT == MVT::i64 ? X86::NEG64r : X86::NEG32r;
287 if (AM.IndexReg.getNode())
290 Index = CurDAG->getRegister(0, VT);
295 Disp = CurDAG->getTargetGlobalAddress(AM.GV,
SDLoc(),
299 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment,
300 AM.Disp, AM.SymbolFlags);
302 assert(!AM.Disp &&
"Non-zero displacement is ignored with ES.");
303 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
304 }
else if (AM.MCSym) {
305 assert(!AM.Disp &&
"Non-zero displacement is ignored with MCSym.");
306 assert(AM.SymbolFlags == 0 &&
"oo");
307 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32);
308 }
else if (AM.JT != -1) {
309 assert(!AM.Disp &&
"Non-zero displacement is ignored with JT.");
310 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
311 }
else if (AM.BlockAddr)
312 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
315 Disp = CurDAG->getTargetConstant(AM.Disp,
DL, MVT::i32);
317 if (AM.Segment.getNode())
318 Segment = AM.Segment;
320 Segment = CurDAG->getRegister(0, MVT::i16);
329 bool shouldAvoidImmediateInstFormsForSize(
SDNode *
N)
const {
335 if (!CurDAG->shouldOptForSize())
345 if (
User->isMachineOpcode()) {
368 auto *
C = dyn_cast<ConstantSDNode>(
N);
369 if (
C && isInt<8>(
C->getSExtValue()))
389 (RegNode = dyn_cast_or_null<RegisterSDNode>(
391 if ((RegNode->
getReg() == X86::ESP) ||
392 (RegNode->
getReg() == X86::RSP))
401 return (UseCount > 1);
406 return CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
411 return CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
416 return CurDAG->getTargetConstant(Imm,
DL, MVT::i64);
421 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
423 MVT VecVT =
N->getOperand(0).getSimpleValueType();
429 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
431 MVT VecVT =
N->getSimpleValueType(0);
435 SDValue getPermuteVINSERTCommutedImmediate(
SDNode *
N,
unsigned VecWidth,
437 assert(VecWidth == 128 &&
"Unexpected vector width");
439 MVT VecVT =
N->getSimpleValueType(0);
441 assert((InsertIdx == 0 || InsertIdx == 1) &&
"Bad insertf128 index");
444 return getI8Imm(InsertIdx ? 0x02 : 0x30,
DL);
449 MVT VT =
N->getSimpleValueType(0);
452 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
454 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
455 if (VT == MVT::i64) {
457 CurDAG->getMachineNode(
458 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
459 CurDAG->getTargetConstant(0, dl, MVT::i64), Zero,
460 CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)),
465 unsigned Opcode =
N->getOpcode();
467 "Unexpected opcode for SBB materialization");
468 unsigned FlagOpIndex = Opcode ==
X86ISD::SBB ? 2 : 1;
470 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
471 N->getOperand(FlagOpIndex),
SDValue());
475 unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr;
476 MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
477 VTs = CurDAG->getVTList(SBBVT, MVT::i32);
479 CurDAG->getMachineNode(Opc, dl, VTs,
480 {Zero, Zero, EFLAGS, EFLAGS.getValue(1)}),
486 bool isUnneededShiftMask(
SDNode *
N,
unsigned Width)
const {
488 const APInt &Val = cast<ConstantSDNode>(
N->getOperand(1))->getAPIntValue();
493 APInt Mask = Val | CurDAG->computeKnownBits(
N->getOperand(0)).Zero;
500 SDNode *getGlobalBaseReg();
511 return Subtarget->getInstrInfo();
524 bool isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const;
528 if (!
N->isNonTemporal())
531 unsigned StoreSize =
N->getMemoryVT().getStoreSize();
533 if (
N->getAlign().value() < StoreSize)
542 return Subtarget->hasSSE41();
544 return Subtarget->hasAVX2();
546 return Subtarget->hasAVX512();
550 bool foldLoadStoreIntoMemOperand(
SDNode *
Node);
553 bool shrinkAndImmediate(
SDNode *
N);
554 bool isMaskZeroExtended(
SDNode *
N)
const;
555 bool tryShiftAmountMod(
SDNode *
N);
556 bool tryShrinkShlLogicImm(
SDNode *
N);
562 bool tryMatchBitSelect(
SDNode *
N);
564 MachineSDNode *emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
566 MachineSDNode *emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
570 bool tryOptimizeRem8Extend(
SDNode *
N);
572 bool onlyUsesZeroFlag(
SDValue Flags)
const;
573 bool hasNoSignFlagUses(
SDValue Flags)
const;
574 bool hasNoCarryFlagUses(
SDValue Flags)
const;
578char X86DAGToDAGISel::ID = 0;
585 unsigned Opcode =
N->getOpcode();
592 EVT OpVT =
N->getOperand(0).getValueType();
596 OpVT =
N->getOperand(1).getValueType();
598 return Subtarget->hasVLX();
612bool X86DAGToDAGISel::isMaskZeroExtended(
SDNode *
N)
const {
634 if (useNonTemporalLoad(cast<LoadSDNode>(
N)))
639 switch (
U->getOpcode()) {
665 if (
auto *Imm = dyn_cast<ConstantSDNode>(Op1)) {
666 if (
Imm->getAPIntValue().isSignedIntN(8))
675 Imm->getAPIntValue().getBitWidth() == 64 &&
676 Imm->getAPIntValue().isIntN(32))
683 (
Imm->getAPIntValue() == UINT8_MAX ||
684 Imm->getAPIntValue() == UINT16_MAX ||
685 Imm->getAPIntValue() == UINT32_MAX))
691 (-
Imm->getAPIntValue()).isSignedIntN(8))
695 (-
Imm->getAPIntValue()).isSignedIntN(8) &&
696 hasNoCarryFlagUses(
SDValue(U, 1)))
721 if (
U->getOperand(0).getOpcode() ==
ISD::SHL &&
725 if (
U->getOperand(1).getOpcode() ==
ISD::SHL &&
733 auto *
C = dyn_cast<ConstantSDNode>(U0.
getOperand(0));
734 if (
C &&
C->getSExtValue() == -2)
739 auto *
C = dyn_cast<ConstantSDNode>(U1.
getOperand(0));
740 if (
C &&
C->getSExtValue() == -2)
754 if (isa<ConstantSDNode>(
U->getOperand(1)))
775bool X86DAGToDAGISel::isProfitableToFormMaskedOp(
SDNode *
N)
const {
778 "Unexpected opcode!");
783 return N->getOperand(1).hasOneUse();
792 if (Chain.
getNode() == Load.getNode())
796 "Unexpected chain operand");
810 Load.getOperand(1), Load.getOperand(2));
814 Ops.
append(Call->op_begin() + 1, Call->op_end());
830 auto *LD = dyn_cast<LoadSDNode>(
Callee.getNode());
848 if (isa<MemSDNode>(Chain.
getNode()) &&
849 cast<MemSDNode>(Chain.
getNode())->writeMem())
855 Callee.getValue(1).hasOneUse())
863 if ((Imm & 0x00FFFFFF) != 0x0F1EFA)
866 uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64,
867 0x65, 0x66, 0x67, 0xf0, 0xf2};
870 uint8_t Byte = (Imm >> i) & 0xFF;
881void X86DAGToDAGISel::PreprocessISelDAG() {
882 bool MadeChange =
false;
884 E = CurDAG->allnodes_end();
I !=
E; ) {
903 MVT VT =
N->getSimpleValueType(0);
904 int64_t
Imm = cast<ConstantSDNode>(
N)->getSExtValue();
905 int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB;
912 SDValue Complement = CurDAG->getConstant(~Imm, dl, VT,
false,
true);
913 Complement = CurDAG->getNOT(dl, Complement, VT);
915 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Complement);
925 if (
N->getOpcode() ==
X86ISD::AND && !
N->hasAnyUseOfValue(1)) {
927 N->getOperand(0),
N->getOperand(1));
929 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
953 auto mayPreventLoadFold = [&]() {
955 N->getOpcode() ==
ISD::ADD && Subtarget->hasAVX() &&
956 !
N->getOperand(1).hasOneUse();
959 N->getSimpleValueType(0).isVector() && !mayPreventLoadFold()) {
965 MVT VT =
N->getSimpleValueType(0);
973 CurDAG->getNode(NewOpcode,
DL, VT,
N->getOperand(0),
AllOnes);
975 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
982 switch (
N->getOpcode()) {
984 MVT VT =
N->getSimpleValueType(0);
986 if (!Subtarget->hasBWI() && (VT == MVT::v32i16 || VT == MVT::v64i8)) {
987 MVT NarrowVT = VT == MVT::v32i16 ? MVT::v16i16 : MVT::v32i8;
993 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
994 unsigned Index = VT == MVT::v32i16 ? 16 : 32;
996 CurDAG->getIntPtrConstant(
Index, dl));
999 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1008 MVT VT =
N->getSimpleValueType(0);
1010 if (!Subtarget->hasBWI() && (VT == MVT::v32i16 || VT == MVT::v64i8)) {
1011 MVT NarrowVT = VT == MVT::v32i16 ? MVT::v16i16 : MVT::v32i8;
1012 auto *MemNode = cast<MemSDNode>(
N);
1014 SDVTList VTs = CurDAG->getVTList(NarrowVT, MVT::Other);
1015 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()};
1016 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode(
1018 MemNode->getMemOperand());
1021 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1022 unsigned Index = VT == MVT::v32i16 ? 16 : 32;
1024 CurDAG->getIntPtrConstant(
Index, dl));
1028 CurDAG->ReplaceAllUsesWith(
N, To);
1038 EVT EleVT =
N->getOperand(0).getValueType().getVectorElementType();
1039 if (EleVT == MVT::i1)
1042 assert(Subtarget->hasSSE41() &&
"Expected SSE4.1 support!");
1043 assert(
N->getValueType(0).getVectorElementType() != MVT::i16 &&
1044 "We can't replace VSELECT with BLENDV in vXi16!");
1046 if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(
N->getOperand(0)) ==
1049 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
1050 CurDAG->getTargetConstant(0xCA,
SDLoc(
N), MVT::i8));
1053 N->getOperand(0),
N->getOperand(1),
1057 CurDAG->ReplaceAllUsesWith(
N,
R.getNode());
1070 if (!
N->getSimpleValueType(0).isVector())
1074 switch (
N->getOpcode()) {
1084 if (
N->isStrictFPOpcode())
1086 CurDAG->getNode(NewOpc,
SDLoc(
N), {
N->getValueType(0), MVT::Other},
1087 {
N->getOperand(0),
N->getOperand(1)});
1090 CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1093 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1103 if (!
N->getValueType(0).isVector())
1107 switch (
N->getOpcode()) {
1113 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1114 N->getOperand(0),
N->getOperand(1));
1116 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1125 if (!
N->getValueType(0).isVector())
1129 if (
N->getOperand(0).getScalarValueSizeInBits() == 1) {
1131 "Unexpected opcode for mask vector!");
1139 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1142 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1162 switch (
N->getOpcode()) {
1178 bool IsStrict =
N->isStrictFPOpcode();
1182 {
N->getValueType(0), MVT::Other},
1183 {
N->getOperand(0),
N->getOperand(1),
1184 CurDAG->getTargetConstant(Imm, dl, MVT::i32)});
1188 CurDAG->getTargetConstant(Imm, dl, MVT::i32));
1190 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1201 MVT VT =
N->getSimpleValueType(0);
1202 if (VT.
isVector() || VT == MVT::f128)
1205 MVT VecVT = VT == MVT::f64 ? MVT::v2f64
1206 : VT == MVT::f32 ? MVT::v4f32
1216 if (Subtarget->hasSSE2()) {
1221 switch (
N->getOpcode()) {
1228 Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1);
1231 Res = CurDAG->getNode(
N->getOpcode(), dl, VecVT, Op0, Op1);
1234 CurDAG->getIntPtrConstant(0, dl));
1236 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1246 !Subtarget->useIndirectThunkCalls() &&
1247 ((
N->getOpcode() ==
X86ISD::CALL && !Subtarget->slowTwoMemOps()) ||
1249 (Subtarget->is64Bit() ||
1250 !getTargetMachine().isPositionIndependent())))) {
1289 switch (
N->getOpcode()) {
1294 MVT SrcVT =
N->getOperand(0).getSimpleValueType();
1295 MVT DstVT =
N->getSimpleValueType(0);
1307 if (SrcIsSSE && DstIsSSE)
1310 if (!SrcIsSSE && !DstIsSSE) {
1315 if (
N->getConstantOperandVal(1))
1323 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1324 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1332 CurDAG->getEntryNode(), dl,
N->getOperand(0), MemTmp, MPI, MemVT);
1334 MemTmp, MPI, MemVT);
1341 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Result);
1350 MVT SrcVT =
N->getOperand(1).getSimpleValueType();
1351 MVT DstVT =
N->getSimpleValueType(0);
1363 if (SrcIsSSE && DstIsSSE)
1366 if (!SrcIsSSE && !DstIsSSE) {
1371 if (
N->getConstantOperandVal(2))
1379 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1380 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1390 SDVTList VTs = CurDAG->getVTList(MVT::Other);
1391 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), MemTmp};
1395 if (
N->getFlags().hasNoFPExcept()) {
1397 Flags.setNoFPExcept(
true);
1398 Store->setFlags(Flags);
1401 assert(SrcVT == MemVT &&
"Unexpected VT!");
1402 Store = CurDAG->getStore(
N->getOperand(0), dl,
N->getOperand(1), MemTmp,
1407 SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other);
1409 Result = CurDAG->getMemIntrinsicNode(
1412 if (
N->getFlags().hasNoFPExcept()) {
1414 Flags.setNoFPExcept(
true);
1418 assert(DstVT == MemVT &&
"Unexpected VT!");
1419 Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI);
1427 CurDAG->ReplaceAllUsesWith(
N,
Result.getNode());
1441 CurDAG->RemoveDeadNodes();
1445bool X86DAGToDAGISel::tryOptimizeRem8Extend(
SDNode *
N) {
1446 unsigned Opc =
N->getMachineOpcode();
1447 if (Opc != X86::MOVZX32rr8 && Opc != X86::MOVSX32rr8 &&
1448 Opc != X86::MOVSX64rr8)
1460 unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX
1461 : X86::MOVSX32rr8_NOREX;
1466 if (Opc == X86::MOVSX64rr8) {
1471 ReplaceUses(
N, Extend);
1480void X86DAGToDAGISel::PostprocessISelDAG() {
1487 bool MadeChange =
false;
1488 while (Position != CurDAG->allnodes_begin()) {
1491 if (
N->use_empty() || !
N->isMachineOpcode())
1494 if (tryOptimizeRem8Extend(
N)) {
1501 unsigned Opc =
N->getMachineOpcode();
1502 if ((Opc == X86::TEST8rr || Opc == X86::TEST16rr ||
1503 Opc == X86::TEST32rr || Opc == X86::TEST64rr) &&
1504 N->getOperand(0) ==
N->getOperand(1) &&
1505 N->getOperand(0)->hasNUsesOfValue(2,
N->getOperand(0).getResNo()) &&
1506 N->getOperand(0).isMachineOpcode()) {
1508 unsigned N0Opc =
And.getMachineOpcode();
1509 if ((N0Opc == X86::AND8rr || N0Opc == X86::AND16rr ||
1510 N0Opc == X86::AND32rr || N0Opc == X86::AND64rr) &&
1511 !
And->hasAnyUseOfValue(1)) {
1516 ReplaceUses(
N,
Test);
1520 if ((N0Opc == X86::AND8rm || N0Opc == X86::AND16rm ||
1521 N0Opc == X86::AND32rm || N0Opc == X86::AND64rm) &&
1522 !
And->hasAnyUseOfValue(1)) {
1525 case X86::AND8rm: NewOpc = X86::TEST8mr;
break;
1526 case X86::AND16rm: NewOpc = X86::TEST16mr;
break;
1527 case X86::AND32rm: NewOpc = X86::TEST32mr;
break;
1528 case X86::AND64rm: NewOpc = X86::TEST64mr;
break;
1538 And.getOperand(6) };
1540 MVT::i32, MVT::Other, Ops);
1541 CurDAG->setNodeMemRefs(
1542 Test, cast<MachineSDNode>(
And.getNode())->memoperands());
1554 if ((Opc == X86::KORTESTBrr || Opc == X86::KORTESTWrr ||
1555 Opc == X86::KORTESTDrr || Opc == X86::KORTESTQrr) &&
1556 N->getOperand(0) ==
N->getOperand(1) &&
1557 N->isOnlyUserOf(
N->getOperand(0).getNode()) &&
1558 N->getOperand(0).isMachineOpcode() &&
1561 unsigned N0Opc =
And.getMachineOpcode();
1564 if (N0Opc == X86::KANDBrr ||
1565 (N0Opc == X86::KANDWrr && Subtarget->hasDQI()) ||
1566 N0Opc == X86::KANDDrr || N0Opc == X86::KANDQrr) {
1570 case X86::KORTESTBrr: NewOpc = X86::KTESTBrr;
break;
1571 case X86::KORTESTWrr: NewOpc = X86::KTESTWrr;
break;
1572 case X86::KORTESTDrr: NewOpc = X86::KTESTDrr;
break;
1573 case X86::KORTESTQrr: NewOpc = X86::KTESTQrr;
break;
1579 ReplaceUses(
N, KTest);
1586 if (Opc != TargetOpcode::SUBREG_TO_REG)
1589 unsigned SubRegIdx =
N->getConstantOperandVal(2);
1590 if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm)
1601 case X86::VMOVAPDrr:
case X86::VMOVUPDrr:
1602 case X86::VMOVAPSrr:
case X86::VMOVUPSrr:
1603 case X86::VMOVDQArr:
case X86::VMOVDQUrr:
1604 case X86::VMOVAPDYrr:
case X86::VMOVUPDYrr:
1605 case X86::VMOVAPSYrr:
case X86::VMOVUPSYrr:
1606 case X86::VMOVDQAYrr:
case X86::VMOVDQUYrr:
1607 case X86::VMOVAPDZ128rr:
case X86::VMOVUPDZ128rr:
1608 case X86::VMOVAPSZ128rr:
case X86::VMOVUPSZ128rr:
1609 case X86::VMOVDQA32Z128rr:
case X86::VMOVDQU32Z128rr:
1610 case X86::VMOVDQA64Z128rr:
case X86::VMOVDQU64Z128rr:
1611 case X86::VMOVAPDZ256rr:
case X86::VMOVUPDZ256rr:
1612 case X86::VMOVAPSZ256rr:
case X86::VMOVUPSZ256rr:
1613 case X86::VMOVDQA32Z256rr:
case X86::VMOVDQU32Z256rr:
1614 case X86::VMOVDQA64Z256rr:
case X86::VMOVDQU64Z256rr:
1619 if (!
In.isMachineOpcode() ||
1620 In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END)
1633 CurDAG->UpdateNodeOperands(
N,
N->getOperand(0), In,
N->getOperand(2));
1638 CurDAG->RemoveDeadNodes();
1643void X86DAGToDAGISel::emitSpecialCodeForMain() {
1644 if (Subtarget->isTargetCygMing()) {
1646 auto &
DL = CurDAG->getDataLayout();
1649 CLI.setChain(CurDAG->getRoot())
1651 CurDAG->getExternalSymbol(
"__main", TLI->getPointerTy(
DL)),
1655 CurDAG->setRoot(
Result.second);
1659void X86DAGToDAGISel::emitFunctionEntryCode() {
1662 if (
F.hasExternalLinkage() &&
F.getName() ==
"main")
1663 emitSpecialCodeForMain();
1673 return isInt<31>(Val);
1677 X86ISelAddressMode &AM) {
1682 int64_t Val = AM.Disp +
Offset;
1685 if (Val != 0 && (AM.ES || AM.MCSym))
1689 if (Subtarget->is64Bit()) {
1692 AM.hasSymbolicDisplacement()))
1696 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
1705bool X86DAGToDAGISel::matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
1706 bool AllowSegmentRegForX32) {
1718 if (
isNullConstant(Address) && AM.Segment.getNode() ==
nullptr &&
1719 !IndirectTlsSegRefs &&
1720 (Subtarget->isTargetGlibc() || Subtarget->isTargetAndroid() ||
1721 Subtarget->isTargetFuchsia())) {
1722 if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32)
1724 switch (
N->getPointerInfo().getAddrSpace()) {
1726 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
1729 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
1742bool X86DAGToDAGISel::matchWrapper(
SDValue N, X86ISelAddressMode &AM) {
1745 if (AM.hasSymbolicDisplacement())
1748 bool IsRIPRelTLS =
false;
1762 if (Subtarget->is64Bit() &&
1768 if (IsRIPRel && AM.hasBaseOrIndexReg())
1772 X86ISelAddressMode Backup = AM;
1776 if (
auto *
G = dyn_cast<GlobalAddressSDNode>(N0)) {
1777 AM.GV =
G->getGlobal();
1778 AM.SymbolFlags =
G->getTargetFlags();
1780 }
else if (
auto *CP = dyn_cast<ConstantPoolSDNode>(N0)) {
1781 AM.CP =
CP->getConstVal();
1782 AM.Alignment =
CP->getAlign();
1783 AM.SymbolFlags =
CP->getTargetFlags();
1785 }
else if (
auto *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
1786 AM.ES = S->getSymbol();
1787 AM.SymbolFlags = S->getTargetFlags();
1788 }
else if (
auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
1789 AM.MCSym = S->getMCSymbol();
1790 }
else if (
auto *J = dyn_cast<JumpTableSDNode>(N0)) {
1791 AM.JT = J->getIndex();
1792 AM.SymbolFlags = J->getTargetFlags();
1793 }
else if (
auto *BA = dyn_cast<BlockAddressSDNode>(N0)) {
1794 AM.BlockAddr = BA->getBlockAddress();
1795 AM.SymbolFlags = BA->getTargetFlags();
1796 Offset = BA->getOffset();
1800 if (foldOffsetIntoAddress(
Offset, AM)) {
1806 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64));
1814bool X86DAGToDAGISel::matchAddress(
SDValue N, X86ISelAddressMode &AM) {
1815 if (matchAddressRecursively(
N, AM, 0))
1822 if (Subtarget->isTarget64BitILP32() &&
1823 AM.BaseType == X86ISelAddressMode::RegBase &&
1824 AM.Base_Reg.getNode() !=
nullptr && AM.IndexReg.getNode() ==
nullptr) {
1825 SDValue Save_Base_Reg = AM.Base_Reg;
1826 if (
auto *LoadN = dyn_cast<LoadSDNode>(Save_Base_Reg)) {
1828 if (matchLoadInAddress(LoadN, AM,
true))
1829 AM.Base_Reg = Save_Base_Reg;
1835 if (AM.Scale == 2 &&
1836 AM.BaseType == X86ISelAddressMode::RegBase &&
1837 AM.Base_Reg.getNode() ==
nullptr) {
1838 AM.Base_Reg = AM.IndexReg;
1845 switch (
TM.getCodeModel()) {
1849 if (Subtarget->is64Bit() &&
1851 AM.BaseType == X86ISelAddressMode::RegBase &&
1852 AM.Base_Reg.getNode() ==
nullptr &&
1853 AM.IndexReg.getNode() ==
nullptr &&
1855 AM.hasSymbolicDisplacement())
1856 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64);
1863bool X86DAGToDAGISel::matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
1869 X86ISelAddressMode Backup = AM;
1870 if (!matchAddressRecursively(
N.getOperand(0), AM,
Depth+1) &&
1871 !matchAddressRecursively(Handle.getValue().getOperand(1), AM,
Depth+1))
1876 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM,
1878 !matchAddressRecursively(Handle.getValue().getOperand(0), AM,
Depth + 1))
1885 if (AM.BaseType == X86ISelAddressMode::RegBase &&
1886 !AM.Base_Reg.getNode() &&
1887 !AM.IndexReg.getNode()) {
1888 N = Handle.getValue();
1889 AM.Base_Reg =
N.getOperand(0);
1890 AM.IndexReg =
N.getOperand(1);
1894 N = Handle.getValue();
1904 if (
N->getNodeId() == -1 ||
1924 X86ISelAddressMode &AM) {
1931 if (ScaleLog <= 0 || ScaleLog >= 4 ||
1932 Mask != (0xffu << ScaleLog))
1935 MVT VT =
N.getSimpleValueType();
1958 AM.Scale = (1 << ScaleLog);
1966 X86ISelAddressMode &AM) {
1972 int64_t Mask = cast<ConstantSDNode>(
N->getOperand(1))->getSExtValue();
1977 bool FoundAnyExtend =
false;
1981 FoundAnyExtend =
true;
1999 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3)
2002 MVT VT =
N.getSimpleValueType();
2004 if (FoundAnyExtend) {
2025 AM.Scale = 1 << ShiftAmt;
2026 AM.IndexReg = NewAnd;
2060 X86ISelAddressMode &AM) {
2071 unsigned AMShiftAmt = MaskTZ;
2075 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2083 unsigned ScaleDown = (64 -
X.getSimpleValueType().getSizeInBits()) + ShiftAmt;
2084 if (MaskLZ < ScaleDown)
2086 MaskLZ -= ScaleDown;
2094 bool ReplacingAnyExtend =
false;
2096 unsigned ExtendBits =
X.getSimpleValueType().getSizeInBits() -
2097 X.getOperand(0).getSimpleValueType().getSizeInBits();
2100 X =
X.getOperand(0);
2101 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits;
2102 ReplacingAnyExtend =
true;
2104 APInt MaskedHighBits =
2107 if (MaskedHighBits != Known.
Zero)
return true;
2111 MVT VT =
N.getSimpleValueType();
2112 if (ReplacingAnyExtend) {
2113 assert(
X.getValueType() != VT);
2137 AM.Scale = 1 << AMShiftAmt;
2138 AM.IndexReg = NewSRL;
2148 X86ISelAddressMode &AM,
2156 if (!Subtarget.hasTBM() &&
2157 !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR()))
2171 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2173 MVT VT =
N.getSimpleValueType();
2196 AM.Scale = 1 << AMShiftAmt;
2197 AM.IndexReg = NewAnd;
2201bool X86DAGToDAGISel::matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
2205 dbgs() <<
"MatchAddress: ";
2210 return matchAddressBase(
N, AM);
2215 if (AM.isRIPRelative()) {
2219 if (!(AM.ES || AM.MCSym) && AM.JT != -1)
2222 if (
auto *Cst = dyn_cast<ConstantSDNode>(
N))
2223 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM))
2228 switch (
N.getOpcode()) {
2231 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
2232 if (
const auto *ESNode = dyn_cast<MCSymbolSDNode>(
N.getOperand(0))) {
2234 AM.MCSym = ESNode->getMCSymbol();
2240 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2241 if (!foldOffsetIntoAddress(Val, AM))
2248 if (!matchWrapper(
N, AM))
2253 if (!matchLoadInAddress(cast<LoadSDNode>(
N), AM))
2258 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2259 AM.Base_Reg.getNode() ==
nullptr &&
2261 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
2262 AM.Base_FrameIndex = cast<FrameIndexSDNode>(
N)->getIndex();
2268 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2271 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
2272 unsigned Val = CN->getZExtValue();
2277 if (Val == 1 || Val == 2 || Val == 3) {
2278 AM.Scale = 1 << Val;
2284 if (CurDAG->isBaseWithConstantOffset(ShVal)) {
2286 auto *AddVal = cast<ConstantSDNode>(ShVal.
getOperand(1));
2288 if (!foldOffsetIntoAddress(Disp, AM))
2292 AM.IndexReg = ShVal;
2300 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2304 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2305 "Unexpected value size!");
2314 if (!isa<ConstantSDNode>(
N.getOperand(1)) ||
2315 !isa<ConstantSDNode>(
And.getOperand(1)))
2317 uint64_t Mask =
And.getConstantOperandVal(1) >>
N.getConstantOperandVal(1);
2329 if (
N.getResNo() != 0)
break;
2334 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2335 AM.Base_Reg.getNode() ==
nullptr &&
2336 AM.IndexReg.getNode() ==
nullptr) {
2337 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
2338 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
2339 CN->getZExtValue() == 9) {
2340 AM.Scale =
unsigned(CN->getZExtValue())-1;
2351 auto *AddVal = cast<ConstantSDNode>(MulVal.
getOperand(1));
2352 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue();
2353 if (foldOffsetIntoAddress(Disp, AM))
2354 Reg =
N.getOperand(0);
2356 Reg =
N.getOperand(0);
2359 AM.IndexReg = AM.Base_Reg =
Reg;
2378 X86ISelAddressMode Backup = AM;
2379 if (matchAddressRecursively(
N.getOperand(0), AM,
Depth+1)) {
2380 N = Handle.getValue();
2384 N = Handle.getValue();
2386 if (AM.IndexReg.getNode() || AM.isRIPRelative()) {
2401 RHS.getOperand(0).getValueType() == MVT::i32))
2405 if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() &&
2406 !AM.Base_Reg.getNode()->hasOneUse()) ||
2407 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
2411 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
2412 ((AM.Disp != 0) && (Backup.Disp == 0)) +
2413 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
2425 AM.NegateIndex =
true;
2431 if (!matchAdd(
N, AM,
Depth))
2442 if (CurDAG->haveNoCommonBitsSet(
N.getOperand(0),
N.getOperand(1)) &&
2460 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2464 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2465 "Unexpected value size!");
2467 if (!isa<ConstantSDNode>(
N.getOperand(1)))
2470 if (
N.getOperand(0).getOpcode() ==
ISD::SRL) {
2499 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2505 if (Src.getOpcode() ==
ISD::AND && Src.hasOneUse())
2506 if (
auto *MaskC = dyn_cast<ConstantSDNode>(Src.getOperand(1))) {
2507 Mask = MaskC->getAPIntValue();
2508 Src = Src.getOperand(0);
2511 if (Src.getOpcode() !=
ISD::SHL || !Src.hasOneUse())
2515 SDValue ShlSrc = Src.getOperand(0);
2516 SDValue ShlAmt = Src.getOperand(1);
2517 auto *ShAmtC = dyn_cast<ConstantSDNode>(ShlAmt);
2520 unsigned ShAmtV = ShAmtC->getZExtValue();
2528 if (!CurDAG->MaskedValueIsZero(ShlSrc, HighZeros & Mask))
2536 MVT VT =
N.getSimpleValueType();
2540 if (!
Mask.isAllOnes()) {
2541 Res = CurDAG->getConstant(
Mask.lshr(ShAmtV),
DL, SrcVT);
2543 Res = CurDAG->getNode(
ISD::AND,
DL, SrcVT, ShlSrc, Res);
2552 AM.Scale = 1 << ShAmtV;
2555 CurDAG->ReplaceAllUsesWith(
N, NewShl);
2556 CurDAG->RemoveDeadNode(
N.getNode());
2561 return matchAddressBase(
N, AM);
2566bool X86DAGToDAGISel::matchAddressBase(
SDValue N, X86ISelAddressMode &AM) {
2568 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
2570 if (!AM.IndexReg.getNode()) {
2581 AM.BaseType = X86ISelAddressMode::RegBase;
2586bool X86DAGToDAGISel::matchVectorAddressRecursively(
SDValue N,
2587 X86ISelAddressMode &AM,
2591 dbgs() <<
"MatchVectorAddress: ";
2596 return matchAddressBase(
N, AM);
2599 switch (
N.getOpcode()) {
2601 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2602 if (!foldOffsetIntoAddress(Val, AM))
2607 if (!matchWrapper(
N, AM))
2615 X86ISelAddressMode Backup = AM;
2616 if (!matchVectorAddressRecursively(
N.getOperand(0), AM,
Depth + 1) &&
2617 !matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2623 if (!matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2625 !matchVectorAddressRecursively(Handle.getValue().getOperand(0), AM,
2630 N = Handle.getValue();
2635 return matchAddressBase(
N, AM);
2641bool X86DAGToDAGISel::matchVectorAddress(
SDValue N, X86ISelAddressMode &AM) {
2642 return matchVectorAddressRecursively(
N, AM, 0);
2650 X86ISelAddressMode AM;
2651 AM.IndexReg = IndexOp;
2652 AM.Scale = cast<ConstantSDNode>(ScaleOp)->getZExtValue();
2656 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2658 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2660 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2666 if (matchVectorAddress(BasePtr, AM))
2669 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2683 X86ISelAddressMode AM;
2695 unsigned AddrSpace =
2696 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
2698 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2700 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2702 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2707 MVT VT =
N.getSimpleValueType();
2709 if (matchAddress(
N, AM))
2712 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2722 N =
N.getOperand(0);
2733 std::optional<ConstantRange> CR =
2734 cast<GlobalAddressSDNode>(
N)->getGlobal()->getAbsoluteSymbolRange();
2738 return CR->getUnsignedMax().ult(1ull << 32);
2747 if (!selectLEAAddr(
N,
Base, Scale,
Index, Disp, Segment))
2750 auto *
RN = dyn_cast<RegisterSDNode>(
Base);
2751 if (RN &&
RN->getReg() == 0)
2752 Base = CurDAG->getRegister(0, MVT::i64);
2753 else if (
Base.getValueType() == MVT::i32 && !isa<FrameIndexSDNode>(
Base)) {
2757 Base = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
2761 RN = dyn_cast<RegisterSDNode>(
Index);
2762 if (RN &&
RN->getReg() == 0)
2763 Index = CurDAG->getRegister(0, MVT::i64);
2766 "Expect to be extending 32-bit registers for use in LEA");
2769 Index = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
2778bool X86DAGToDAGISel::selectLEAAddr(
SDValue N,
2782 X86ISelAddressMode AM;
2786 MVT VT =
N.getSimpleValueType();
2791 SDValue T = CurDAG->getRegister(0, MVT::i32);
2793 if (matchAddress(
N, AM))
2798 unsigned Complexity = 0;
2799 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode())
2801 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
2804 if (AM.IndexReg.getNode())
2817 if (AM.hasSymbolicDisplacement()) {
2819 if (Subtarget->is64Bit())
2829 auto isMathWithFlags = [](
SDValue V) {
2830 switch (
V.getOpcode()) {
2851 if (isMathWithFlags(
N.getOperand(0)) || isMathWithFlags(
N.getOperand(1)))
2859 if (Complexity <= 2)
2862 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2871 auto *GA = cast<GlobalAddressSDNode>(
N);
2873 X86ISelAddressMode AM;
2874 AM.GV = GA->getGlobal();
2875 AM.Disp += GA->getOffset();
2876 AM.SymbolFlags = GA->getTargetFlags();
2878 if (Subtarget->is32Bit()) {
2880 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
2883 MVT VT =
N.getSimpleValueType();
2884 getAddressOperands(AM,
SDLoc(
N), VT,
Base, Scale,
Index, Disp, Segment);
2892 EVT VT =
N.getValueType();
2893 bool WasTruncated =
false;
2895 WasTruncated =
true;
2896 N =
N.getOperand(0);
2905 unsigned Opc =
N.getOperand(0)->getOpcode();
2907 Op =
N.getOperand(0);
2910 return !WasTruncated;
2914 auto *GA = cast<GlobalAddressSDNode>(
N.getOperand(0));
2915 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange();
2916 if (!CR || CR->getUnsignedMax().uge(1ull << VT.
getSizeInBits()))
2920 Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(),
SDLoc(
N), VT,
2921 GA->getOffset(), GA->getTargetFlags());
2929 assert(Root &&
P &&
"Unknown root/parent nodes");
2931 !IsProfitableToFold(
N,
P, Root) ||
2932 !IsLegalToFold(
N,
P, Root, OptLevel))
2935 return selectAddr(
N.getNode(),
2936 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
2943 assert(Root &&
P &&
"Unknown root/parent nodes");
2945 !IsProfitableToFold(
N,
P, Root) ||
2946 !IsLegalToFold(
N,
P, Root, OptLevel))
2949 return selectAddr(
N.getNode(),
2950 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
2956SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
2957 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
2959 return CurDAG->getRegister(GlobalBaseReg, TLI->
getPointerTy(
DL)).getNode();
2962bool X86DAGToDAGISel::isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const {
2964 N =
N->getOperand(0).getNode();
2968 auto *GA = dyn_cast<GlobalAddressSDNode>(
N->getOperand(0));
2972 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange();
2976 return CR->getSignedMin().sge(-1ull << Width) &&
2977 CR->getSignedMax().slt(1ull << Width);
2981 assert(
N->isMachineOpcode() &&
"Unexpected node");
2982 unsigned Opc =
N->getMachineOpcode();
2983 const MCInstrDesc &MCID = getInstrInfo()->get(Opc);
2988 return static_cast<X86::CondCode>(
N->getConstantOperandVal(CondNo));
2993bool X86DAGToDAGISel::onlyUsesZeroFlag(
SDValue Flags)
const {
2998 if (UI.getUse().getResNo() !=
Flags.getResNo())
3002 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3006 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3008 if (FlagUI.getUse().getResNo() != 1)
continue;
3010 if (!FlagUI->isMachineOpcode())
return false;
3029bool X86DAGToDAGISel::hasNoSignFlagUses(
SDValue Flags)
const {
3034 if (UI.getUse().getResNo() !=
Flags.getResNo())
3038 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3042 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3044 if (FlagUI.getUse().getResNo() != 1)
continue;
3046 if (!FlagUI->isMachineOpcode())
return false;
3085 bool X86DAGToDAGISel::hasNoCarryFlagUses(
SDValue Flags)
const {
3090 if (UI.getUse().getResNo() !=
Flags.getResNo())
3093 unsigned UIOpc = UI->getOpcode();
3097 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3101 FlagUI != FlagUE; ++FlagUI) {
3103 if (FlagUI.getUse().getResNo() != 1)
3106 if (!FlagUI->isMachineOpcode())
3147 if (StoredVal.
getResNo() != 0)
return false;
3161 LoadNode = cast<LoadSDNode>(Load);
3164 if (!Load.hasOneUse())
3172 bool FoundLoad =
false;
3176 const unsigned int Max = 1024;
3218 if (Chain == Load.getValue(1)) {
3224 if (Op == Load.getValue(1)) {
3240 if (Op.getNode() != LoadNode)
3273bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(
SDNode *
Node) {
3274 auto *StoreNode = cast<StoreSDNode>(
Node);
3281 EVT MemVT = StoreNode->getMemoryVT();
3282 if (MemVT != MVT::i64 && MemVT != MVT::i32 && MemVT != MVT::i16 &&
3286 bool IsCommutable =
false;
3287 bool IsNegate =
false;
3301 IsCommutable =
true;
3305 unsigned LoadOpNo = IsNegate ? 1 : 0;
3309 LoadNode, InputChain)) {
3316 LoadNode, InputChain))
3325 auto SelectOpcode = [&](
unsigned Opc64,
unsigned Opc32,
unsigned Opc16,
3346 unsigned NewOpc = SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m,
3356 if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) {
3360 if ((IsOne || IsNegOne) && hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3363 ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
3364 : SelectOpcode(
X86::DEC64m,
X86::DEC32m,
X86::DEC16m,
X86::DEC8m);
3377 auto SelectRegOpcode = [SelectOpcode](
unsigned Opc) {
3380 return SelectOpcode(X86::ADD64mr, X86::ADD32mr, X86::ADD16mr,
3383 return SelectOpcode(X86::ADC64mr, X86::ADC32mr, X86::ADC16mr,
3386 return SelectOpcode(X86::SUB64mr, X86::SUB32mr, X86::SUB16mr,
3389 return SelectOpcode(X86::SBB64mr, X86::SBB32mr, X86::SBB16mr,
3392 return SelectOpcode(X86::AND64mr, X86::AND32mr, X86::AND16mr,
3395 return SelectOpcode(X86::OR64mr, X86::OR32mr, X86::OR16mr, X86::OR8mr);
3397 return SelectOpcode(X86::XOR64mr, X86::XOR32mr, X86::XOR16mr,
3403 auto SelectImmOpcode = [SelectOpcode](
unsigned Opc) {
3406 return SelectOpcode(X86::ADD64mi32, X86::ADD32mi, X86::ADD16mi,
3409 return SelectOpcode(X86::ADC64mi32, X86::ADC32mi, X86::ADC16mi,
3412 return SelectOpcode(X86::SUB64mi32, X86::SUB32mi, X86::SUB16mi,
3415 return SelectOpcode(X86::SBB64mi32, X86::SBB32mi, X86::SBB16mi,
3418 return SelectOpcode(X86::AND64mi32, X86::AND32mi, X86::AND16mi,
3421 return SelectOpcode(X86::OR64mi32, X86::OR32mi, X86::OR16mi,
3424 return SelectOpcode(X86::XOR64mi32, X86::XOR32mi, X86::XOR16mi,
3431 unsigned NewOpc = SelectRegOpcode(Opc);
3436 if (
auto *OperandC = dyn_cast<ConstantSDNode>(Operand)) {
3437 int64_t OperandV = OperandC->getSExtValue();
3443 ((MemVT != MVT::i8 && !isInt<8>(OperandV) && isInt<8>(-OperandV)) ||
3444 (MemVT == MVT::i64 && !isInt<32>(OperandV) &&
3445 isInt<32>(-OperandV))) &&
3446 hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3447 OperandV = -OperandV;
3451 if (MemVT != MVT::i64 || isInt<32>(OperandV)) {
3452 Operand = CurDAG->getTargetConstant(OperandV,
SDLoc(
Node), MemVT);
3453 NewOpc = SelectImmOpcode(Opc);
3459 CurDAG->getCopyToReg(InputChain,
SDLoc(
Node), X86::EFLAGS,
3463 Segment, Operand, CopyTo, CopyTo.
getValue(1)};
3464 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3468 Segment, Operand, InputChain};
3469 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3480 CurDAG->setNodeMemRefs(Result, MemOps);
3486 CurDAG->RemoveDeadNode(
Node);
3496bool X86DAGToDAGISel::matchBitExtract(
SDNode *
Node) {
3499 "Should be either an and-mask, or right-shift after clearing high bits.");
3502 if (!Subtarget->hasBMI() && !Subtarget->hasBMI2())
3505 MVT NVT =
Node->getSimpleValueType(0);
3508 if (NVT != MVT::i32 && NVT != MVT::i64)
3516 const bool AllowExtraUsesByDefault = Subtarget->hasBMI2();
3517 auto checkUses = [AllowExtraUsesByDefault](
3519 std::optional<bool> AllowExtraUses) {
3520 return AllowExtraUses.value_or(AllowExtraUsesByDefault) ||
3521 Op.getNode()->hasNUsesOfValue(NUses,
Op.getResNo());
3523 auto checkOneUse = [checkUses](
SDValue Op,
3524 std::optional<bool> AllowExtraUses =
3526 return checkUses(Op, 1, AllowExtraUses);
3528 auto checkTwoUse = [checkUses](
SDValue Op,
3529 std::optional<bool> AllowExtraUses =
3531 return checkUses(Op, 2, AllowExtraUses);
3534 auto peekThroughOneUseTruncation = [checkOneUse](
SDValue V) {
3536 assert(
V.getSimpleValueType() == MVT::i32 &&
3537 V.getOperand(0).getSimpleValueType() == MVT::i64 &&
3538 "Expected i64 -> i32 truncation");
3539 V =
V.getOperand(0);
3545 auto matchPatternA = [checkOneUse, peekThroughOneUseTruncation, &NBits,
3548 if (
Mask->getOpcode() !=
ISD::ADD || !checkOneUse(Mask))
3554 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3559 NBits =
M0->getOperand(1);
3560 NegateNBits =
false;
3564 auto isAllOnes = [
this, peekThroughOneUseTruncation, NVT](
SDValue V) {
3565 V = peekThroughOneUseTruncation(V);
3566 return CurDAG->MaskedValueIsAllOnes(
3572 auto matchPatternB = [checkOneUse, isAllOnes, peekThroughOneUseTruncation,
3575 if (
Mask.getOpcode() !=
ISD::XOR || !checkOneUse(Mask))
3578 if (!isAllOnes(
Mask->getOperand(1)))
3581 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3585 if (!isAllOnes(
M0->getOperand(0)))
3587 NBits =
M0->getOperand(1);
3588 NegateNBits =
false;
3594 auto canonicalizeShiftAmt = [&NBits, &NegateNBits](
SDValue ShiftAmt,
3595 unsigned Bitwidth) {
3600 NBits = NBits.getOperand(0);
3605 auto *V0 = dyn_cast<ConstantSDNode>(NBits.getOperand(0));
3606 if (!V0 || V0->getZExtValue() != Bitwidth)
3608 NBits = NBits.getOperand(1);
3609 NegateNBits =
false;
3615 auto matchPatternC = [checkOneUse, peekThroughOneUseTruncation, &NegateNBits,
3618 Mask = peekThroughOneUseTruncation(Mask);
3619 unsigned Bitwidth =
Mask.getSimpleValueType().getSizeInBits();
3621 if (
Mask.getOpcode() !=
ISD::SRL || !checkOneUse(Mask))
3628 if (!checkOneUse(
M1))
3630 canonicalizeShiftAmt(
M1, Bitwidth);
3635 return !NegateNBits;
3643 auto matchPatternD = [checkOneUse, checkTwoUse, canonicalizeShiftAmt,
3644 AllowExtraUsesByDefault, &NegateNBits,
3657 canonicalizeShiftAmt(N1, Bitwidth);
3661 const bool AllowExtraUses = AllowExtraUsesByDefault && !NegateNBits;
3662 if (!checkOneUse(N0, AllowExtraUses) || !checkTwoUse(N1, AllowExtraUses))
3668 auto matchLowBitMask = [matchPatternA, matchPatternB,
3670 return matchPatternA(Mask) || matchPatternB(Mask) || matchPatternC(Mask);
3674 X =
Node->getOperand(0);
3677 if (matchLowBitMask(Mask)) {
3681 if (!matchLowBitMask(Mask))
3684 }
else if (!matchPatternD(
Node))
3689 if (NegateNBits && !Subtarget->hasBMI2())
3701 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, MVT::i32), 0);
3704 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit,
DL, MVT::i32);
3706 NBits =
SDValue(CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG,
DL,
3707 MVT::i32, ImplDef, NBits, SRIdxVal),
3717 NBits = CurDAG->getNode(
ISD::SUB,
DL, MVT::i32, BitWidthC, NBits);
3721 if (Subtarget->hasBMI2()) {
3723 if (NVT != MVT::i32) {
3731 SelectCode(Extract.
getNode());
3740 SDValue RealX = peekThroughOneUseTruncation(
X);
3746 MVT XVT =
X.getSimpleValueType();
3756 SDValue C8 = CurDAG->getConstant(8,
DL, MVT::i8);
3764 SDValue ShiftAmt =
X.getOperand(1);
3765 X =
X.getOperand(0);
3768 "Expected shift amount to be i8");
3772 SDValue OrigShiftAmt = ShiftAmt;
3777 Control = CurDAG->getNode(
ISD::OR,
DL, MVT::i32, Control, ShiftAmt);
3782 if (XVT != MVT::i32) {
3797 SelectCode(Extract.
getNode());
3804 MVT NVT =
Node->getSimpleValueType(0);
3817 Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR());
3818 if (!PreferBEXTR && !Subtarget->hasBMI2())
3830 if (NVT != MVT::i32 && NVT != MVT::i64)
3834 auto *MaskCst = dyn_cast<ConstantSDNode>(N1);
3835 auto *ShiftCst = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
3836 if (!MaskCst || !ShiftCst)
3844 uint64_t Shift = ShiftCst->getZExtValue();
3849 if (Shift == 8 && MaskSize == 8)
3860 if (!PreferBEXTR && MaskSize <= 32)
3864 unsigned ROpc, MOpc;
3867 assert(Subtarget->hasBMI2() &&
"We must have BMI2's BZHI then.");
3871 Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT);
3872 ROpc = NVT == MVT::i64 ? X86::BZHI64rr : X86::BZHI32rr;
3873 MOpc = NVT == MVT::i64 ? X86::BZHI64rm : X86::BZHI32rm;
3874 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
3875 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
3881 Control = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT);
3882 if (Subtarget->hasTBM()) {
3883 ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri;
3884 MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi;
3886 assert(Subtarget->hasBMI() &&
"We must have BMI1's BEXTR then.");
3888 ROpc = NVT == MVT::i64 ? X86::BEXTR64rr : X86::BEXTR32rr;
3889 MOpc = NVT == MVT::i64 ? X86::BEXTR64rm : X86::BEXTR32rm;
3890 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
3891 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
3897 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
3898 if (tryFoldLoad(
Node, N0.
getNode(), Input, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
3900 Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Control, Input.
getOperand(0)};
3901 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
3902 NewNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
3906 CurDAG->setNodeMemRefs(NewNode, {cast<LoadSDNode>(Input)->getMemOperand()});
3908 NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, Control);
3913 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT);
3914 unsigned NewOpc = NVT == MVT::i64 ? X86::SHR64ri : X86::SHR32ri;
3916 CurDAG->getMachineNode(NewOpc, dl, NVT,
SDValue(NewNode, 0), ShAmt);
3923MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
3924 bool MayFoldLoad,
const SDLoc &dl,
3929 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
3930 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
3933 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
3934 if (MayFoldLoad && tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
3935 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
3937 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other);
3938 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
3942 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
3947 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32);
3948 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
3955MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
3956 bool MayFoldLoad,
const SDLoc &dl,
3962 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
3963 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
3966 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
3967 if (MayFoldLoad && tryFoldLoad(
Node, N2, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
3968 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
3970 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other, MVT::Glue);
3971 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
3976 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N2)->getMemOperand()});
3981 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Glue);
3982 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
3987bool X86DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
3988 EVT VT =
N->getValueType(0);
3995 unsigned Size = VT == MVT::i64 ? 64 : 32;
3997 SDValue OrigShiftAmt =
N->getOperand(1);
3998 SDValue ShiftAmt = OrigShiftAmt;
4013 auto *Add0C = dyn_cast<ConstantSDNode>(Add0);
4014 auto *Add1C = dyn_cast<ConstantSDNode>(Add1);
4017 if (Add1C && Add1C->getAPIntValue().urem(
Size) == 0) {
4021 ((Add0C && Add0C->getAPIntValue().urem(
Size) ==
Size - 1) ||
4022 (Add1C && Add1C->getAPIntValue().urem(
Size) ==
Size - 1))) {
4026 assert(Add0C ==
nullptr || Add1C ==
nullptr);
4035 NewShiftAmt = CurDAG->getNode(
ISD::XOR,
DL, OpVT,
4036 Add0C ==
nullptr ? Add0 : Add1,
AllOnes);
4042 Add0C->getZExtValue() != 0) {
4045 if (Add0C->getZExtValue() %
Size == 0)
4048 Add0C->getZExtValue() % 32 == 0) {
4056 Add0 = CurDAG->getZExtOrTrunc(Add0,
DL, SubVT);
4060 X = CurDAG->getNode(
ISD::ADD,
DL, SubVT, Add1, Add0);
4082 NewShiftAmt = CurDAG->getNode(
ISD::TRUNCATE,
DL, MVT::i8, NewShiftAmt);
4089 NewShiftAmt = CurDAG->getNode(
ISD::AND,
DL, MVT::i8, NewShiftAmt,
4090 CurDAG->getConstant(
Size - 1,
DL, MVT::i8));
4094 SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(
N,
N->getOperand(0),
4096 if (UpdatedNode !=
N) {
4099 ReplaceNode(
N, UpdatedNode);
4106 CurDAG->RemoveDeadNode(OrigShiftAmt.
getNode());
4114bool X86DAGToDAGISel::tryShrinkShlLogicImm(
SDNode *
N) {
4115 MVT NVT =
N->getSimpleValueType(0);
4116 unsigned Opcode =
N->getOpcode();
4124 auto *Cst = dyn_cast<ConstantSDNode>(N1);
4128 int64_t Val = Cst->getSExtValue();
4133 bool FoundAnyExtend =
false;
4137 FoundAnyExtend =
true;
4145 if (NVT != MVT::i32 && NVT != MVT::i64)
4148 auto *ShlCst = dyn_cast<ConstantSDNode>(Shift.
getOperand(1));
4152 uint64_t ShAmt = ShlCst->getZExtValue();
4156 uint64_t RemovedBitsMask = (1ULL << ShAmt) - 1;
4157 if (Opcode !=
ISD::AND && (Val & RemovedBitsMask) != 0)
4162 auto CanShrinkImmediate = [&](int64_t &ShiftedVal) {
4166 ShiftedVal = (
uint64_t)Val >> ShAmt;
4167 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4170 if (ShiftedVal == UINT8_MAX || ShiftedVal == UINT16_MAX)
4173 ShiftedVal = Val >> ShAmt;
4174 if ((!isInt<8>(Val) && isInt<8>(ShiftedVal)) ||
4175 (!isInt<32>(Val) && isInt<32>(ShiftedVal)))
4179 ShiftedVal = (
uint64_t)Val >> ShAmt;
4180 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4187 if (!CanShrinkImmediate(ShiftedVal))
4197 unsigned ZExtWidth = Cst->getAPIntValue().getActiveBits();
4203 NeededMask &= ~Cst->getAPIntValue();
4205 if (CurDAG->MaskedValueIsZero(
N->getOperand(0), NeededMask))
4210 if (FoundAnyExtend) {
4216 SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT);
4218 SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT,
X, NewCst);
4227bool X86DAGToDAGISel::matchVPTERNLOG(
SDNode *Root,
SDNode *ParentA,
4231 assert(
A.isOperandOf(ParentA) &&
B.isOperandOf(ParentB) &&
4232 C.isOperandOf(ParentC) &&
"Incorrect parent node");
4234 auto tryFoldLoadOrBCast =
4237 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4243 L =
L.getOperand(0);
4250 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4251 unsigned Size = MemIntr->getMemoryVT().getSizeInBits();
4255 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4258 bool FoldedLoad =
false;
4259 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4260 if (tryFoldLoadOrBCast(Root, ParentC,
C, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4262 }
else if (tryFoldLoadOrBCast(Root, ParentA,
A, Tmp0, Tmp1, Tmp2, Tmp3,
4267 uint8_t OldImm =
Imm;
4268 Imm = OldImm & 0xa5;
4269 if (OldImm & 0x02)
Imm |= 0x10;
4270 if (OldImm & 0x10)
Imm |= 0x02;
4271 if (OldImm & 0x08)
Imm |= 0x40;
4272 if (OldImm & 0x40)
Imm |= 0x08;
4273 }
else if (tryFoldLoadOrBCast(Root, ParentB,
B, Tmp0, Tmp1, Tmp2, Tmp3,
4278 uint8_t OldImm =
Imm;
4279 Imm = OldImm & 0x99;
4280 if (OldImm & 0x02)
Imm |= 0x04;
4281 if (OldImm & 0x04)
Imm |= 0x02;
4282 if (OldImm & 0x20)
Imm |= 0x40;
4283 if (OldImm & 0x40)
Imm |= 0x20;
4288 SDValue TImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
4294 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
4298 auto *MemIntr = cast<MemIntrinsicSDNode>(
C);
4299 unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits();
4300 assert((EltSize == 32 || EltSize == 64) &&
"Unexpected broadcast size!");
4302 bool UseD = EltSize == 32;
4304 Opc = UseD ? X86::VPTERNLOGDZ128rmbi : X86::VPTERNLOGQZ128rmbi;
4306 Opc = UseD ? X86::VPTERNLOGDZ256rmbi : X86::VPTERNLOGQZ256rmbi;
4308 Opc = UseD ? X86::VPTERNLOGDZrmbi : X86::VPTERNLOGQZrmbi;
4314 Opc = UseD ? X86::VPTERNLOGDZ128rmi : X86::VPTERNLOGQZ128rmi;
4316 Opc = UseD ? X86::VPTERNLOGDZ256rmi : X86::VPTERNLOGQZ256rmi;
4318 Opc = UseD ? X86::VPTERNLOGDZrmi : X86::VPTERNLOGQZrmi;
4323 SDValue Ops[] = {
A,
B, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, TImm,
C.getOperand(0)};
4324 MNode = CurDAG->getMachineNode(Opc,
DL, VTs, Ops);
4327 ReplaceUses(
C.getValue(1),
SDValue(MNode, 1));
4329 CurDAG->setNodeMemRefs(MNode, {cast<MemSDNode>(
C)->getMemOperand()});
4334 Opc = UseD ? X86::VPTERNLOGDZ128rri : X86::VPTERNLOGQZ128rri;
4336 Opc = UseD ? X86::VPTERNLOGDZ256rri : X86::VPTERNLOGQZ256rri;
4338 Opc = UseD ? X86::VPTERNLOGDZrri : X86::VPTERNLOGQZrri;
4342 MNode = CurDAG->getMachineNode(Opc,
DL, NVT, {
A,
B,
C, TImm});
4346 CurDAG->RemoveDeadNode(Root);
4352bool X86DAGToDAGISel::tryVPTERNLOG(
SDNode *
N) {
4353 MVT NVT =
N->getSimpleValueType(0);
4356 if (!NVT.
isVector() || !Subtarget->hasAVX512() ||
4367 auto getFoldableLogicOp = [](
SDValue Op) {
4370 Op =
Op.getOperand(0);
4372 if (!
Op.hasOneUse())
4375 unsigned Opc =
Op.getOpcode();
4384 if ((FoldableOp = getFoldableLogicOp(N1))) {
4386 }
else if ((FoldableOp = getFoldableLogicOp(N0))) {
4399 uint8_t TernlogMagicA = 0xf0;
4400 uint8_t TernlogMagicB = 0xcc;
4401 uint8_t TernlogMagicC = 0xaa;
4410 Parent =
Op.getNode();
4411 Op =
Op.getOperand(0);
4415 PeekThroughNot(
A, ParentA, TernlogMagicA);
4416 PeekThroughNot(
B, ParentB, TernlogMagicB);
4417 PeekThroughNot(
C, ParentC, TernlogMagicC);
4422 case ISD::AND:
Imm = TernlogMagicB & TernlogMagicC;
break;
4423 case ISD::OR:
Imm = TernlogMagicB | TernlogMagicC;
break;
4424 case ISD::XOR:
Imm = TernlogMagicB ^ TernlogMagicC;
break;
4428 switch (
N->getOpcode()) {
4432 Imm &= ~TernlogMagicA;
4434 Imm = ~(
Imm) & TernlogMagicA;
4441 return matchVPTERNLOG(
N, ParentA, ParentB, ParentC,
A,
B,
C, Imm);
4451bool X86DAGToDAGISel::shrinkAndImmediate(
SDNode *
And) {
4454 MVT VT =
And->getSimpleValueType(0);
4455 if (VT != MVT::i32 && VT != MVT::i64)
4458 auto *And1C = dyn_cast<ConstantSDNode>(
And->getOperand(1));
4467 APInt MaskVal = And1C->getAPIntValue();
4469 if (!MaskLZ || (VT == MVT::i64 && MaskLZ == 32))
4473 if (VT == MVT::i64 && MaskLZ >= 32) {
4475 MaskVal = MaskVal.
trunc(32);
4480 APInt NegMaskVal = MaskVal | HighZeros;
4489 if (VT == MVT::i64 && MaskVal.
getBitWidth() < 64) {
4490 NegMaskVal = NegMaskVal.
zext(64);
4491 HighZeros = HighZeros.
zext(64);
4496 if (!CurDAG->MaskedValueIsZero(And0, HighZeros))
4516 bool FoldedBCast,
bool Masked) {
4517#define VPTESTM_CASE(VT, SUFFIX) \
4520 return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
4521 return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
4524#define VPTESTM_BROADCAST_CASES(SUFFIX) \
4525default: llvm_unreachable("Unexpected VT!"); \
4526VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
4527VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
4528VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
4529VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
4530VPTESTM_CASE(v16i32, DZ##SUFFIX) \
4531VPTESTM_CASE(v8i64, QZ##SUFFIX)
4533#define VPTESTM_FULL_CASES(SUFFIX) \
4534VPTESTM_BROADCAST_CASES(SUFFIX) \
4535VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
4536VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
4537VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
4538VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
4539VPTESTM_CASE(v64i8, BZ##SUFFIX) \
4540VPTESTM_CASE(v32i16, WZ##SUFFIX)
4558#undef VPTESTM_FULL_CASES
4559#undef VPTESTM_BROADCAST_CASES
4565bool X86DAGToDAGISel::tryVPTESTM(
SDNode *Root,
SDValue Setcc,
4567 assert(Subtarget->hasAVX512() &&
"Expected AVX512!");
4617 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4622 if (CmpSVT != MVT::i32 && CmpSVT != MVT::i64)
4628 L =
L.getOperand(0);
4634 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4635 if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.
getSizeInBits())
4638 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4642 bool CanFoldLoads = Src0 != Src1;
4644 bool FoldedLoad =
false;
4645 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4647 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src1, Tmp0, Tmp1, Tmp2,
4651 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src0, Tmp0, Tmp1,
4660 bool IsMasked = InMask.
getNode() !=
nullptr;
4673 SDValue ImplDef =
SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl,
4675 Src0 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src0);
4678 Src1 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src1);
4683 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4684 InMask =
SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4685 dl, MaskVT, InMask, RC), 0);
4690 unsigned Opc =
getVPTESTMOpc(CmpVT, IsTestN, FoldedLoad, FoldedBCast,
4695 SDVTList VTs = CurDAG->getVTList(MaskVT, MVT::Other);
4698 SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4700 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
4702 SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4704 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
4710 CurDAG->setNodeMemRefs(CNode, {cast<MemSDNode>(Src1)->getMemOperand()});
4713 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, InMask, Src0, Src1);
4715 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, Src0, Src1);
4721 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4722 CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4723 dl, ResVT,
SDValue(CNode, 0), RC);
4727 CurDAG->RemoveDeadNode(Root);
4733bool X86DAGToDAGISel::tryMatchBitSelect(
SDNode *
N) {
4736 MVT NVT =
N->getSimpleValueType(0);
4739 if (!NVT.
isVector() || !Subtarget->hasAVX512())
4773 SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8);
4782 MVT NVT =
Node->getSimpleValueType(0);
4783 unsigned Opcode =
Node->getOpcode();
4786 if (
Node->isMachineOpcode()) {
4788 Node->setNodeId(-1);
4795 unsigned IntNo =
Node->getConstantOperandVal(1);
4798 case Intrinsic::x86_encodekey128:
4799 case Intrinsic::x86_encodekey256: {
4800 if (!Subtarget->hasKL())
4806 case Intrinsic::x86_encodekey128: Opcode = X86::ENCODEKEY128;
break;
4807 case Intrinsic::x86_encodekey256: Opcode = X86::ENCODEKEY256;
break;
4811 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(3),
4813 if (Opcode == X86::ENCODEKEY256)
4814 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(4),
4818 Opcode, dl,
Node->getVTList(),
4819 {Node->getOperand(2), Chain, Chain.getValue(1)});
4820 ReplaceNode(
Node, Res);
4823 case Intrinsic::x86_tileloadd64_internal:
4824 case Intrinsic::x86_tileloaddt164_internal: {
4825 if (!Subtarget->hasAMXTILE())
4827 unsigned Opc = IntNo == Intrinsic::x86_tileloadd64_internal
4829 : X86::PTILELOADDT1V;
4832 SDValue Scale = getI8Imm(1, dl);
4834 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
4835 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
4839 Node->getOperand(3),
4846 CNode = CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops);
4847 ReplaceNode(
Node, CNode);
4854 unsigned IntNo =
Node->getConstantOperandVal(1);
4857 case Intrinsic::x86_sse3_monitor:
4858 case Intrinsic::x86_monitorx:
4859 case Intrinsic::x86_clzero: {
4860 bool Use64BitPtr =
Node->getOperand(2).getValueType() == MVT::i64;
4865 case Intrinsic::x86_sse3_monitor:
4866 if (!Subtarget->hasSSE3())
4868 Opc = Use64BitPtr ? X86::MONITOR64rrr : X86::MONITOR32rrr;
4870 case Intrinsic::x86_monitorx:
4871 if (!Subtarget->hasMWAITX())
4873 Opc = Use64BitPtr ? X86::MONITORX64rrr : X86::MONITORX32rrr;
4875 case Intrinsic::x86_clzero:
4876 if (!Subtarget->hasCLZERO())
4878 Opc = Use64BitPtr ? X86::CLZERO64r : X86::CLZERO32r;
4883 unsigned PtrReg = Use64BitPtr ? X86::RAX : X86::EAX;
4884 SDValue Chain = CurDAG->getCopyToReg(
Node->getOperand(0), dl, PtrReg,
4888 if (IntNo == Intrinsic::x86_sse3_monitor ||
4889 IntNo == Intrinsic::x86_monitorx) {
4891 Chain = CurDAG->getCopyToReg(Chain, dl, X86::ECX,
Node->getOperand(3),
4894 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EDX,
Node->getOperand(4),
4899 MachineSDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
4901 ReplaceNode(
Node, CNode);
4907 case Intrinsic::x86_tilestored64_internal: {
4908 unsigned Opc = X86::PTILESTOREDV;
4911 SDValue Scale = getI8Imm(1, dl);
4913 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
4914 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
4918 Node->getOperand(3),