71 #define DEBUG_TYPE "x86-isel"
73 STATISTIC(NumTailCalls,
"Number of tail calls");
76 "x86-experimental-pref-innermost-loop-alignment",
cl::init(4),
78 "Sets the preferable loop alignment for experiments (as log2 bytes) "
79 "for innermost loops only. If specified, this option overrides "
80 "alignment set by x86-experimental-pref-loop-alignment."),
84 "mul-constant-optimization",
cl::init(
true),
85 cl::desc(
"Replace 'mul x, Const' with more effective instructions like "
90 "x86-experimental-unordered-atomic-isel",
cl::init(
false),
91 cl::desc(
"Use LoadSDNode and StoreSDNode instead of "
92 "AtomicSDNode for unordered atomic loads and "
93 "stores respectively."),
110 bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();
123 if (Subtarget.isAtom())
125 else if (Subtarget.is64Bit())
134 if (Subtarget.hasSlowDivide32())
136 if (Subtarget.hasSlowDivide64() && Subtarget.is64Bit())
142 static const struct {
144 const char *
const Name;
154 for (
const auto &LC : LibraryCalls) {
177 if (Subtarget.is64Bit())
203 if (Subtarget.is64Bit())
211 if (Subtarget.is64Bit())
222 if (Subtarget.is64Bit())
226 if (!Subtarget.useSoftFloat()) {
290 if (!Subtarget.is64Bit()) {
303 if (Subtarget.is64Bit()) {
317 if (Subtarget.is64Bit()) {
322 }
else if (!Subtarget.is64Bit())
351 if (Subtarget.is64Bit())
362 if (!Subtarget.useSoftFloat() && Subtarget.hasX87()) {
372 if (Subtarget.hasBMI()) {
381 if (Subtarget.is64Bit()) {
387 if (Subtarget.hasLZCNT()) {
394 if (VT ==
MVT::i64 && !Subtarget.is64Bit())
408 (!Subtarget.useSoftFloat() && Subtarget.hasF16C()) ?
Custom :
Expand);
428 if (Subtarget.is64Bit())
430 if (Subtarget.hasPOPCNT()) {
439 if (Subtarget.is64Bit())
447 if (!Subtarget.hasMOVBE())
458 if (VT ==
MVT::i64 && !Subtarget.is64Bit())
479 if (VT ==
MVT::i64 && !Subtarget.is64Bit())
491 if (VT ==
MVT::i64 && !Subtarget.is64Bit())
514 if (!Subtarget.is64Bit())
543 bool Is64Bit = Subtarget.is64Bit();
591 if (!Subtarget.useSoftFloat() && Subtarget.
hasSSE2()) {
595 : &X86::FR16RegClass);
597 : &X86::FR32RegClass);
599 : &X86::FR64RegClass);
673 }
else if (!Subtarget.useSoftFloat() && Subtarget.
hasSSE1() &&
674 (UseX87 || Is64Bit)) {
726 addLegalFPImmediate(
APFloat(+0.0
f));
727 addLegalFPImmediate(
APFloat(+1.0
f));
728 addLegalFPImmediate(
APFloat(-0.0
f));
729 addLegalFPImmediate(
APFloat(-1.0
f));
731 addLegalFPImmediate(
APFloat(+0.0
f));
736 addLegalFPImmediate(
APFloat(+0.0));
737 addLegalFPImmediate(
APFloat(+1.0));
738 addLegalFPImmediate(
APFloat(-0.0));
739 addLegalFPImmediate(
APFloat(-1.0));
741 addLegalFPImmediate(
APFloat(+0.0));
772 addLegalFPImmediate(TmpFlt);
774 addLegalFPImmediate(TmpFlt);
780 addLegalFPImmediate(TmpFlt2);
782 addLegalFPImmediate(TmpFlt2);
819 if (!Subtarget.useSoftFloat() && Subtarget.is64Bit() && Subtarget.
hasSSE1()) {
821 : &X86::VR128RegClass);
972 if (!Subtarget.useSoftFloat() && Subtarget.
hasMMX()) {
977 if (!Subtarget.useSoftFloat() && Subtarget.
hasSSE1()) {
979 : &X86::VR128RegClass);
1000 if (!Subtarget.useSoftFloat() && Subtarget.
hasSSE2()) {
1002 : &X86::VR128RegClass);
1007 : &X86::VR128RegClass);
1009 : &X86::VR128RegClass);
1011 : &X86::VR128RegClass);
1013 : &X86::VR128RegClass);
1015 : &X86::VR128RegClass);
1098 if (VT ==
MVT::v2i64 && !Subtarget.is64Bit())
1208 if (!Subtarget.useSoftFloat() && Subtarget.
hasSSSE3()) {
1225 if (!Subtarget.useSoftFloat() && Subtarget.
hasSSE41()) {
1280 if (Subtarget.is64Bit() && !Subtarget.
hasAVX512()) {
1292 if (!Subtarget.useSoftFloat() && Subtarget.
hasSSE42()) {
1296 if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
1312 if (!Subtarget.useSoftFloat() && Subtarget.
hasAVX()) {
1316 : &X86::VR256RegClass);
1318 : &X86::VR256RegClass);
1320 : &X86::VR256RegClass);
1322 : &X86::VR256RegClass);
1324 : &X86::VR256RegClass);
1326 : &X86::VR256RegClass);
1328 : &X86::VR256RegClass);
1556 if (!Subtarget.useSoftFloat() && !Subtarget.hasFP16() &&
1557 Subtarget.hasF16C()) {
1578 if (!Subtarget.useSoftFloat() && Subtarget.
hasAVX512()) {
1603 if (!Subtarget.hasDQI()) {
1647 if (!Subtarget.useSoftFloat() && Subtarget.
useAVX512Regs()) {
1648 bool HasBWI = Subtarget.hasBWI();
1716 if (!Subtarget.hasVLX()) {
1836 if (Subtarget.hasDQI()) {
1844 if (Subtarget.hasCDI()) {
1851 if (Subtarget.hasVPOPCNTDQ()) {
1901 if (Subtarget.hasVBMI2()) {
1919 if (!Subtarget.useSoftFloat() && Subtarget.
hasAVX512()) {
1927 if (Subtarget.hasDQI()) {
1932 "Unexpected operation action!");
1961 if (Subtarget.hasDQI()) {
1972 if (Subtarget.hasCDI()) {
1978 if (Subtarget.hasVPOPCNTDQ()) {
1987 if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
2021 if (Subtarget.hasBITALG()) {
2027 if (!Subtarget.useSoftFloat() && Subtarget.hasFP16()) {
2028 auto setGroup = [&] (
MVT VT) {
2131 if (Subtarget.hasVLX()) {
2176 if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) {
2189 if (Subtarget.hasBWI()) {
2194 if (Subtarget.hasFP16()) {
2230 if (Subtarget.hasAMXTILE()) {
2238 if (!Subtarget.is64Bit()) {
2249 if (VT ==
MVT::i64 && !Subtarget.is64Bit())
2267 if (!Subtarget.is64Bit()) {
2305 if (Subtarget.is32Bit() &&
2415 unsigned XorOp = Subtarget.is64Bit() ? X86::XOR64_FP : X86::XOR32_FP;
2423 !Subtarget.hasBWI())
2437 static std::pair<MVT, unsigned>
2464 if (!
isPowerOf2_32(NumElts) || (NumElts == 64 && !Subtarget.hasBWI()) ||
2479 unsigned NumRegisters;
2480 std::tie(RegisterVT, NumRegisters) =
2492 !Subtarget.hasX87())
2506 unsigned NumRegisters;
2507 std::tie(RegisterVT, NumRegisters) =
2510 return NumRegisters;
2519 if (!Subtarget.is64Bit() && !Subtarget.hasX87()) {
2531 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2541 return NumIntermediates;
2549 NumIntermediates = 2;
2554 NumIntermediates, RegisterVT);
2591 if (
VectorType *VTy = dyn_cast<VectorType>(Ty)) {
2592 if (VTy->getPrimitiveSizeInBits().getFixedSize() == 128)
2593 MaxAlign =
Align(16);
2594 }
else if (
ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
2597 if (EltAlign > MaxAlign)
2598 MaxAlign = EltAlign;
2599 }
else if (
StructType *STy = dyn_cast<StructType>(Ty)) {
2600 for (
auto *EltTy : STy->elements()) {
2603 if (EltAlign > MaxAlign)
2604 MaxAlign = EltAlign;
2617 if (Subtarget.is64Bit()) {
2619 Align TyAlign =
DL.getABITypeAlign(Ty);
2621 return TyAlign.
value();
2628 return Alignment.value();
2637 if (!FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat)) {
2638 if (
Op.size() >= 16 &&
2639 (!Subtarget.isUnalignedMem16Slow() ||
Op.isAligned(
Align(16)))) {
2646 if (
Op.size() >= 32 && Subtarget.
hasAVX() &&
2659 if (Subtarget.
hasSSE1() && (Subtarget.is64Bit() || Subtarget.hasX87()) &&
2662 }
else if (((
Op.isMemcpy() && !
Op.isMemcpyStrSrc()) ||
Op.isZeroMemset()) &&
2663 Op.size() >= 8 && !Subtarget.is64Bit() && Subtarget.
hasSSE2()) {
2676 if (Subtarget.is64Bit() &&
Op.size() >= 8)
2699 *Fast = !Subtarget.isUnalignedMem16Slow();
2702 *Fast = !Subtarget.isUnalignedMem32Slow();
2714 return (Alignment < 16 || !Subtarget.
hasSSE41());
2735 return Subtarget.useSoftFloat();
2742 if (Subtarget.is64Bit())
2746 unsigned ParamRegs = 0;
2748 ParamRegs =
M->getNumberRegisterParameters();
2753 if (
T->isIntOrPtrTy())
2755 unsigned numRegs = 1;
2758 if (ParamRegs < numRegs)
2760 ParamRegs -= numRegs;
2780 if (!Subtarget.is64Bit())
2801 std::pair<const TargetRegisterClass *, uint8_t>
2810 RRC = Subtarget.is64Bit() ? &X86::GR64RegClass : &X86::GR32RegClass;
2813 RRC = &X86::VR64RegClass;
2822 RRC = &X86::VR128XRegClass;
2825 return std::make_pair(
RRC, Cost);
2828 unsigned X86TargetLowering::getAddressSpace()
const {
2829 if (Subtarget.is64Bit())
2858 int Offset =
M->getStackProtectorGuardOffset();
2862 if (Offset == INT_MAX)
2863 Offset = (Subtarget.is64Bit()) ? 0x28 : 0x14;
2865 StringRef GuardReg =
M->getStackProtectorGuardReg();
2866 if (GuardReg ==
"fs")
2868 else if (GuardReg ==
"gs")
2872 StringRef GuardSymb =
M->getStackProtectorGuardSymbol();
2873 if (!GuardSymb.
empty()) {
2879 nullptr, GuardSymb,
nullptr,
2896 M.getOrInsertGlobal(
"__security_cookie",
2905 F->addParamAttr(0, Attribute::AttrKind::InReg);
2910 StringRef GuardMode =
M.getStackProtectorGuard();
2913 if ((GuardMode ==
"tls" || GuardMode.
empty()) &&
2923 return M.getGlobalVariable(
"__security_cookie");
2932 return M.getFunction(
"__security_check_cookie");
2948 int Offset = (Subtarget.is64Bit()) ? 0x48 : 0x24;