84#include "llvm/IR/IntrinsicsARM.h"
122#define DEBUG_TYPE "arm-isel"
125STATISTIC(NumMovwMovt,
"Number of GAs materialized with movw + movt");
126STATISTIC(NumLoopByVals,
"Number of loops generated for byval arguments");
128 "Number of constants with their storage promoted into constant pools");
132 cl::desc(
"Enable / disable ARM interworking (for debugging only)"),
137 cl::desc(
"Enable / disable promotion of unnamed_addr constants into "
142 cl::desc(
"Maximum size of constant to promote into a constant pool"),
146 cl::desc(
"Maximum size of ALL constants to promote into a constant pool"),
151 cl::desc(
"Maximum interleave factor for MVE VLDn to generate."),
156 ARM::R0, ARM::R1, ARM::R2, ARM::R3
170void ARMTargetLowering::addTypeForNEON(
MVT VT,
MVT PromotedLdStVT) {
171 if (VT != PromotedLdStVT) {
180 if (ElemTy != MVT::f64)
184 if (ElemTy == MVT::i32) {
228void ARMTargetLowering::addDRTypeForNEON(
MVT VT) {
230 addTypeForNEON(VT, MVT::f64);
233void ARMTargetLowering::addQRTypeForNEON(
MVT VT) {
235 addTypeForNEON(VT, MVT::v2f64);
238void ARMTargetLowering::setAllExpand(
MVT VT) {
251void ARMTargetLowering::addAllExtLoads(
const MVT From,
const MVT To,
252 LegalizeAction Action) {
258void ARMTargetLowering::addMVEVectorTypes(
bool HasMVEFP) {
259 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
261 for (
auto VT : IntTypes) {
335 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
336 for (
auto VT : FloatTypes) {
405 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
406 for (
auto VT : LongTypes) {
423 addAllExtLoads(MVT::v8i16, MVT::v8i8,
Legal);
424 addAllExtLoads(MVT::v4i32, MVT::v4i16,
Legal);
425 addAllExtLoads(MVT::v4i32, MVT::v4i8,
Legal);
442 for (
auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
451 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
452 for (
auto VT : pTypes) {
508 for (
int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID)
516 if (Subtarget->isThumb() && Subtarget->
hasVFP2Base() &&
517 Subtarget->
hasARMOps() && !Subtarget->useSoftFloat()) {
518 static const struct {
520 const char *
const Name;
542 { RTLIB::UO_F32,
"__unordsf2vfp",
ISD::SETNE },
551 { RTLIB::UO_F64,
"__unorddf2vfp",
ISD::SETNE },
576 for (
const auto &LC : LibraryCalls) {
588 static const struct {
590 const char *
const Name;
675 for (
const auto &LC : LibraryCalls) {
685 static const struct {
687 const char *
const Name;
690 } MemOpsLibraryCalls[] = {
698 for (
const auto &LC : MemOpsLibraryCalls) {
708 static const struct {
710 const char *
const Name;
723 for (
const auto &LC : LibraryCalls) {
755 static const struct {
757 const char *
const Name;
765 for (
const auto &LC : LibraryCalls) {
776 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only() &&
777 Subtarget->hasFPRegs()) {
787 setAllExpand(MVT::f32);
788 if (!Subtarget->hasFP64())
789 setAllExpand(MVT::f64);
792 if (Subtarget->hasFullFP16()) {
801 if (Subtarget->hasBF16()) {
803 setAllExpand(MVT::bf16);
804 if (!Subtarget->hasFullFP16())
811 addAllExtLoads(VT, InnerVT,
Expand);
826 if (Subtarget->hasMVEIntegerOps())
827 addMVEVectorTypes(Subtarget->hasMVEFloatOps());
830 if (Subtarget->hasLOB()) {
834 if (Subtarget->hasNEON()) {
835 addDRTypeForNEON(MVT::v2f32);
836 addDRTypeForNEON(MVT::v8i8);
837 addDRTypeForNEON(MVT::v4i16);
838 addDRTypeForNEON(MVT::v2i32);
839 addDRTypeForNEON(MVT::v1i64);
841 addQRTypeForNEON(MVT::v4f32);
842 addQRTypeForNEON(MVT::v2f64);
843 addQRTypeForNEON(MVT::v16i8);
844 addQRTypeForNEON(MVT::v8i16);
845 addQRTypeForNEON(MVT::v4i32);
846 addQRTypeForNEON(MVT::v2i64);
848 if (Subtarget->hasFullFP16()) {
849 addQRTypeForNEON(MVT::v8f16);
850 addDRTypeForNEON(MVT::v4f16);
853 if (Subtarget->hasBF16()) {
854 addQRTypeForNEON(MVT::v8bf16);
855 addDRTypeForNEON(MVT::v4bf16);
859 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
899 if (Subtarget->hasNEON()) {
1013 for (
MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
1022 for (
auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
1031 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
1039 if (Subtarget->hasMVEIntegerOps()) {
1044 if (Subtarget->hasMVEFloatOps()) {
1048 if (!Subtarget->hasFP64()) {
1095 if (Subtarget->hasFullFP16()) {
1101 if (!Subtarget->hasFP16()) {
1149 if (Subtarget->hasDSP()) {
1171 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6Ops()
1172 || (Subtarget->
isThumb2() && !Subtarget->hasDSP()))
1187 if (Subtarget->hasMVEIntegerOps())
1197 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops())
1208 if (!Subtarget->hasV5TOps() || Subtarget->
isThumb1Only()) {
1217 if (Subtarget->hasPerfMon())
1221 if (!Subtarget->hasV6Ops())
1224 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1225 : Subtarget->hasDivideInARMMode();
1232 if (Subtarget->
isTargetWindows() && !Subtarget->hasDivideInThumbMode()) {
1249 HasStandaloneRem =
false;
1254 const char *
const Name;
1256 } LibraryCalls[] = {
1268 for (
const auto &LC : LibraryCalls) {
1275 const char *
const Name;
1277 } LibraryCalls[] = {
1289 for (
const auto &LC : LibraryCalls) {
1327 InsertFencesForAtomic =
false;
1329 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
1333 if (!Subtarget->isThumb() || !Subtarget->
isMClass())
1338 if (!Subtarget->hasAcquireRelease() ||
1341 InsertFencesForAtomic =
true;
1347 if (Subtarget->hasDataBarrier())
1348 InsertFencesForAtomic =
true;
1368 if (!InsertFencesForAtomic) {
1376 (!Subtarget->
isMClass() && Subtarget->hasV6Ops())) {
1388 }
else if ((Subtarget->
isMClass() && Subtarget->hasV8MBaselineOps()) ||
1389 Subtarget->hasForced32BitAtomics()) {
1403 if (!Subtarget->hasV6Ops()) {
1409 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1441 if (Subtarget->hasFullFP16()) {
1451 if (Subtarget->hasFullFP16())
1466 if (!Subtarget->useSoftFloat() && Subtarget->
hasVFP2Base() &&
1480 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only()) {
1488 if (!Subtarget->hasFP16()) {
1519 if (Subtarget->hasNEON()) {
1526 if (Subtarget->hasFP64()) {
1539 if (Subtarget->hasFullFP16()) {
1558 if (Subtarget->hasNEON()) {
1570 if (Subtarget->hasFullFP16()) {
1602 if (Subtarget->hasMVEIntegerOps())
1605 if (Subtarget->hasV6Ops())
1610 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1617 if (Subtarget->useSoftFloat() || Subtarget->
isThumb1Only() ||
1645 return Subtarget->useSoftFloat();
1658std::pair<const TargetRegisterClass *, uint8_t>
1669 case MVT::f32:
case MVT::f64:
case MVT::v8i8:
case MVT::v4i16:
1670 case MVT::v2i32:
case MVT::v1i64:
case MVT::v2f32:
1671 RRC = &ARM::DPRRegClass;
1679 case MVT::v16i8:
case MVT::v8i16:
case MVT::v4i32:
case MVT::v2i64:
1680 case MVT::v4f32:
case MVT::v2f64:
1681 RRC = &ARM::DPRRegClass;
1685 RRC = &ARM::DPRRegClass;
1689 RRC = &ARM::DPRRegClass;
1693 return std::make_pair(RRC,
Cost);
1697#define MAKE_CASE(V) \
1916 if ((Subtarget->hasMVEIntegerOps() &&
1917 (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
1918 VT == MVT::v16i8)) ||
1919 (Subtarget->hasMVEFloatOps() &&
1920 (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16)))
1934 if (Subtarget->hasNEON()) {
1935 if (VT == MVT::v4i64)
1936 return &ARM::QQPRRegClass;
1937 if (VT == MVT::v8i64)
1938 return &ARM::QQQQPRRegClass;
1940 if (Subtarget->hasMVEIntegerOps()) {
1941 if (VT == MVT::v4i64)
1942 return &ARM::MQQPRRegClass;
1943 if (VT == MVT::v8i64)
1944 return &ARM::MQQQQPRRegClass;
1953 Align &PrefAlign)
const {
1954 if (!isa<MemIntrinsic>(CI))
1972 unsigned NumVals =
N->getNumValues();
1976 for (
unsigned i = 0; i != NumVals; ++i) {
1977 EVT VT =
N->getValueType(i);
1978 if (VT == MVT::Glue || VT == MVT::Other)
1984 if (!
N->isMachineOpcode())
2008 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2009 return Const->getZExtValue() == 16;
2016 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2017 return Const->getZExtValue() == 16;
2024 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2025 return Const->getZExtValue() == 16;
2094 bool isVarArg)
const {
2115 else if (Subtarget->hasFPRegs() && !Subtarget->
isThumb1Only() &&
2136 bool isVarArg)
const {
2137 return CCAssignFnForNode(
CC,
false, isVarArg);
2141 bool isVarArg)
const {
2142 return CCAssignFnForNode(
CC,
true, isVarArg);
2149 bool isVarArg)
const {
2150 switch (getEffectiveCallingConv(
CC, isVarArg)) {
2176 if (Subtarget->hasFullFP16()) {
2189 if (Subtarget->hasFullFP16()) {
2203SDValue ARMTargetLowering::LowerCallResult(
2207 SDValue ThisVal,
bool isCmseNSCall)
const {
2215 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
2220 if (i == 0 && isThisReturn) {
2222 "unexpected return calling convention register assignment");
2233 Chain =
Lo.getValue(1);
2234 InGlue =
Lo.getValue(2);
2238 Chain =
Hi.getValue(1);
2239 InGlue =
Hi.getValue(2);
2251 Chain =
Lo.getValue(1);
2252 InGlue =
Lo.getValue(2);
2255 Chain =
Hi.getValue(1);
2256 InGlue =
Hi.getValue(2);
2300std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
2302 bool IsTailCall,
int SPDiff)
const {
2324 return std::make_pair(DstAddr, DstInfo);
2329 RegsToPassVector &RegsToPass,
2336 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
2337 unsigned id = Subtarget->
isLittle() ? 0 : 1;
2350 std::tie(DstAddr, DstInfo) =
2351 computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
2383 bool isStructRet = (Outs.
empty()) ?
false : Outs[0].
Flags.isSRet();
2384 bool isThisReturn =
false;
2385 bool isCmseNSCall =
false;
2386 bool isSibCall =
false;
2387 bool PreferIndirect =
false;
2388 bool GuardWithBTI =
false;
2398 !Subtarget->noBTIAtReturnTwice())
2403 isCmseNSCall =
true;
2415 if (isa<GlobalAddressSDNode>(Callee)) {
2419 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
2422 PreferIndirect = Subtarget->isThumb() && Subtarget->
hasMinSize() &&
2424 return isa<Instruction>(U) &&
2425 cast<Instruction>(U)->getParent() == BB;
2432 IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs, PreferIndirect);
2446 "site marked musttail");
2449 unsigned NumBytes = CCInfo.getStackSize();
2458 if (isTailCall && !isSibCall) {
2465 NumBytes =
alignTo(NumBytes, StackAlign);
2470 SPDiff = NumReusableBytes - NumBytes;
2474 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (
unsigned)-SPDiff)
2490 RegsToPassVector RegsToPass;
2498 bool AfterFormalArgLoads =
false;
2502 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.
size();
2504 ++i, ++realArgIdx) {
2506 SDValue Arg = OutVals[realArgIdx];
2508 bool isByVal =
Flags.isByVal();
2528 if (isTailCall && VA.
isMemLoc() && !AfterFormalArgLoads) {
2530 AfterFormalArgLoads =
true;
2542 auto ArgVT = Outs[realArgIdx].ArgVT;
2543 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2561 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2562 StackPtr, MemOpChains, isTailCall, SPDiff);
2566 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2567 StackPtr, MemOpChains, isTailCall, SPDiff);
2572 std::tie(DstAddr, DstInfo) =
2573 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2577 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2578 StackPtr, MemOpChains, isTailCall, SPDiff);
2580 if (realArgIdx == 0 &&
Flags.isReturned() && !
Flags.isSwiftSelf() &&
2581 Outs[0].VT == MVT::i32) {
2583 "unexpected calling convention register assignment");
2584 assert(!
Ins.empty() && Ins[0].VT == MVT::i32 &&
2585 "unexpected use of 'returned'");
2586 isThisReturn =
true;
2590 CSInfo.ArgRegPairs.emplace_back(VA.
getLocReg(), i);
2591 RegsToPass.push_back(std::make_pair(VA.
getLocReg(), Arg));
2592 }
else if (isByVal) {
2594 unsigned offset = 0;
2598 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2599 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2601 if (CurByValIdx < ByValArgsCount) {
2603 unsigned RegBegin, RegEnd;
2604 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
2609 for (i = 0, j = RegBegin;
j < RegEnd; i++,
j++) {
2616 RegsToPass.push_back(std::make_pair(j, Load));
2621 offset = RegEnd - RegBegin;
2623 CCInfo.nextInRegsParam();
2626 if (
Flags.getByValSize() > 4*offset) {
2630 std::tie(Dst, DstInfo) =
2631 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2640 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2648 std::tie(DstAddr, DstInfo) =
2649 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2656 if (!MemOpChains.
empty())
2662 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
2663 Chain = DAG.
getCopyToReg(Chain, dl, RegsToPass[i].first,
2664 RegsToPass[i].second, InGlue);
2671 bool isDirect =
false;
2676 GVal =
G->getGlobal();
2677 bool isStub = !
TM.shouldAssumeDSOLocal(GVal) && Subtarget->
isTargetMachO();
2679 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->
isMClass());
2680 bool isLocalARMFunc =
false;
2683 if (Subtarget->genLongCalls()) {
2685 "long-calls codegen is not position independent!");
2689 if (isa<GlobalAddressSDNode>(Callee)) {
2690 if (Subtarget->genExecuteOnly()) {
2709 const char *
Sym = S->getSymbol();
2711 if (Subtarget->genExecuteOnly()) {
2730 }
else if (isa<GlobalAddressSDNode>(Callee)) {
2731 if (!PreferIndirect) {
2736 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !
ARMInterworking);
2738 if (isStub && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2750 "Windows is the only supported COFF target");
2754 else if (!
TM.shouldAssumeDSOLocal(GVal))
2770 const char *
Sym = S->getSymbol();
2771 if (isARMFunc && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2775 ARMPCLabelIndex, 4);
2789 assert(!isARMFunc && !isDirect &&
2790 "Cannot handle call to ARM function or direct call");
2793 "call to non-secure function would "
2794 "require passing arguments on stack",
2801 "call to non-secure function would return value through pointer",
2809 if (Subtarget->isThumb()) {
2812 else if (isCmseNSCall)
2814 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2819 if (!isDirect && !Subtarget->hasV5TOps())
2821 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2834 if (isTailCall && !isSibCall) {
2839 std::vector<SDValue> Ops;
2840 Ops.push_back(Chain);
2841 Ops.push_back(Callee);
2849 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2850 Ops.push_back(DAG.
getRegister(RegsToPass[i].first,
2851 RegsToPass[i].second.getValueType()));
2863 isThisReturn =
false;
2869 assert(Mask &&
"Missing call preserved mask for calling convention");
2873 Ops.push_back(InGlue);
2885 Chain = DAG.
getNode(CallOpc, dl, NodeTys, Ops);
2897 Chain = DAG.
getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
2903 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2904 InVals, isThisReturn,
2905 isThisReturn ? OutVals[0] :
SDValue(), isCmseNSCall);
2912void ARMTargetLowering::HandleByVal(
CCState *State,
unsigned &
Size,
2913 Align Alignment)
const {
2915 Alignment = std::max(Alignment,
Align(4));
2921 unsigned AlignInRegs = Alignment.
value() / 4;
2922 unsigned Waste = (ARM::R4 -
Reg) % AlignInRegs;
2923 for (
unsigned i = 0; i < Waste; ++i)
2929 unsigned Excess = 4 * (ARM::R4 -
Reg);
2936 if (NSAAOffset != 0 &&
Size > Excess) {
2948 unsigned ByValRegBegin =
Reg;
2949 unsigned ByValRegEnd = std::min<unsigned>(Reg +
Size / 4, ARM::R4);
2953 for (
unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2959 Size = std::max<int>(
Size - Excess, 0);
2970 int FI = std::numeric_limits<int>::max();
2978 if (!Flags.isByVal()) {
2984 }
else if (
LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
2985 if (Flags.isByVal())
3000 assert(FI != std::numeric_limits<int>::max());
3011bool ARMTargetLowering::IsEligibleForTailCallOptimization(
3033 if (Outs.
size() >= 4 &&
3034 (!isa<GlobalAddressSDNode>(
Callee.getNode()) || isIndirect)) {
3052 return CalleeCC == CallerCC;
3056 bool isCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
3058 if (isCalleeStructRet || isCallerStructRet)
3072 (!
TT.isOSWindows() ||
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO()))
3079 getEffectiveCallingConv(CalleeCC, isVarArg),
3080 getEffectiveCallingConv(CallerCC, CallerF.
isVarArg()), MF,
C, Ins,
3086 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
3087 if (CalleeCC != CallerCC) {
3088 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
3089 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3102 if (!Outs.
empty()) {
3109 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.
size();
3111 ++i, ++realArgIdx) {
3114 SDValue Arg = OutVals[realArgIdx];
3118 if (VA.
needsCustom() && (RegVT == MVT::f64 || RegVT == MVT::v2f64)) {
3125 if (!ArgLocs[++i].isRegLoc())
3127 if (RegVT == MVT::v2f64) {
3128 if (!ArgLocs[++i].isRegLoc())
3130 if (!ArgLocs[++i].isRegLoc())
3155 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
3164 StringRef IntKind =
F.getFnAttribute(
"interrupt").getValueAsString();
3177 if (IntKind ==
"" || IntKind ==
"IRQ" || IntKind ==
"FIQ" ||
3180 else if (IntKind ==
"SWI" || IntKind ==
"UNDEF")
3184 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
3211 bool isLittleEndian = Subtarget->
isLittle();
3223 "secure entry function would return value through pointer",
3229 for (
unsigned i = 0, realRVLocIdx = 0;
3231 ++i, ++realRVLocIdx) {
3235 SDValue Arg = OutVals[realRVLocIdx];
3236 bool ReturnF16 =
false;
3271 auto RetVT = Outs[realRVLocIdx].ArgVT;
3293 DAG.
getVTList(MVT::i32, MVT::i32), Half);
3297 HalfGPRs.
getValue(isLittleEndian ? 0 : 1), Glue);
3303 HalfGPRs.
getValue(isLittleEndian ? 1 : 0), Glue);
3315 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
3317 fmrrd.
getValue(isLittleEndian ? 0 : 1), Glue);
3322 fmrrd.
getValue(isLittleEndian ? 1 : 0), Glue);
3366 return DAG.
getNode(RetNode, dl, MVT::Other, RetOps);
3369bool ARMTargetLowering::isUsedByReturnOnly(
SDNode *
N,
SDValue &Chain)
const {
3370 if (
N->getNumValues() != 1)
3372 if (!
N->hasNUsesOfValue(1, 0))
3380 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3382 TCChain =
Copy->getOperand(0);
3396 SDValue UseChain =
U->getOperand(0);
3404 if (
U->getOperand(
U->getNumOperands() - 1).getValueType() == MVT::Glue)
3412 if (!
Copy->hasOneUse())
3419 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3421 TCChain =
Copy->getOperand(0);
3426 bool HasRet =
false;
3441bool ARMTargetLowering::mayBeEmittedAsTailCall(
const CallInst *CI)
const {
3459 &&
"LowerWRITE_REGISTER called for non-i64 type argument.");
3475 EVT PtrVT =
Op.getValueType();
3485 if (Subtarget->genExecuteOnly()) {
3487 auto T =
const_cast<Type*
>(
CP->getType());
3488 auto C =
const_cast<Constant*
>(
CP->getConstVal());
3499 return LowerGlobalAddress(GA, DAG);
3504 Align CPAlign =
CP->getAlign();
3506 CPAlign = std::max(CPAlign,
Align(4));
3507 if (
CP->isMachineConstantPoolEntry())
3519 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3528 unsigned ARMPCLabelIndex = 0;
3531 const BlockAddress *BA = cast<BlockAddressSDNode>(
Op)->getBlockAddress();
3534 if (!IsPositionIndependent) {
3537 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3548 if (!IsPositionIndependent)
3579ARMTargetLowering::LowerGlobalTLSAddressDarwin(
SDValue Op,
3582 "This function expects a Darwin target");
3587 SDValue DescAddr = LowerGlobalAddressDarwin(
Op, DAG);
3593 MVT::i32,
DL, Chain, DescAddr,
3617 Chain, FuncTLVGet, DAG.
getRegister(ARM::R0, MVT::i32),
3623ARMTargetLowering::LowerGlobalTLSAddressWindows(
SDValue Op,
3640 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
3667 const auto *GA = cast<GlobalAddressSDNode>(
Op);
3684 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3706 Args.push_back(Entry);
3714 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
3715 return CallResult.first;
3737 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3745 PtrVT, dl, Chain,
Offset,
3747 Chain =
Offset.getValue(1);
3753 PtrVT, dl, Chain,
Offset,
3763 PtrVT, dl, Chain,
Offset,
3779 return LowerGlobalTLSAddressDarwin(
Op, DAG);
3782 return LowerGlobalTLSAddressWindows(
Op, DAG);
3791 return LowerToTLSGeneralDynamicModel(GA, DAG);
3794 return LowerToTLSExecModels(GA, DAG, model);
3803 while (!Worklist.
empty()) {
3805 if (isa<ConstantExpr>(U)) {
3810 auto *
I = dyn_cast<Instruction>(U);
3811 if (!
I ||
I->getParent()->getParent() !=
F)
3839 auto *GVar = dyn_cast<GlobalVariable>(GV);
3840 if (!GVar || !GVar->hasInitializer() ||
3841 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3842 !GVar->hasLocalLinkage())
3847 auto *
Init = GVar->getInitializer();
3849 Init->needsDynamicRelocation())
3858 auto *CDAInit = dyn_cast<ConstantDataArray>(
Init);
3861 unsigned RequiredPadding = 4 - (
Size % 4);
3862 bool PaddingPossible =
3863 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3868 unsigned PaddedSize =
Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3892 if (RequiredPadding != 4) {
3897 while (RequiredPadding--)
3909 ++NumConstpoolPromoted;
3914 if (
const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
3915 if (!(GV = GA->getAliaseeObject()))
3917 if (
const auto *V = dyn_cast<GlobalVariable>(GV))
3918 return V->isConstant();
3919 return isa<Function>(GV);
3927 return LowerGlobalAddressWindows(
Op, DAG);
3929 return LowerGlobalAddressELF(
Op, DAG);
3931 return LowerGlobalAddressDarwin(
Op, DAG);
3939 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
3943 if (GV->
isDSOLocal() && !Subtarget->genExecuteOnly())
3956 }
else if (Subtarget->
isROPI() && IsRO) {
3961 }
else if (Subtarget->
isRWPI() && !IsRO) {
3986 if (Subtarget->
useMovt() || Subtarget->genExecuteOnly()) {
4005 "ROPI/RWPI not currently supported for Darwin");
4008 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4031 "Windows on ARM expects to use movw/movt");
4033 "ROPI/RWPI not currently supported for Windows");
4036 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4040 else if (!
TM.shouldAssumeDSOLocal(GV))
4064 DAG.
getVTList(MVT::i32, MVT::Other),
Op.getOperand(0),
4065 Op.getOperand(1), Val);
4082SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
4085 Op.getConstantOperandVal(
Op.getOperand(0).getValueType() == MVT::Other);
4089 case Intrinsic::arm_gnu_eabi_mcount: {
4098 assert(Mask &&
"Missing call preserved mask for calling convention");
4103 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
4107 if (Subtarget->isThumb())
4110 ARM::tBL_PUSHLR, dl, ResultTys,
4111 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
4112 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
4116 {ReturnAddress, Callee, RegisterMask, Chain}),
4125 unsigned IntNo =
Op.getConstantOperandVal(0);
4129 case Intrinsic::thread_pointer: {
4133 case Intrinsic::arm_cls: {
4134 const SDValue &Operand =
Op.getOperand(1);
4135 const EVT VTy =
Op.getValueType();
4146 case Intrinsic::arm_cls64: {
4149 const SDValue &Operand =
Op.getOperand(1);
4150 const EVT VTy =
Op.getValueType();
4173 case Intrinsic::eh_sjlj_lsda: {
4180 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
4190 if (IsPositionIndependent) {
4196 case Intrinsic::arm_neon_vabs:
4199 case Intrinsic::arm_neon_vabds:
4200 if (
Op.getValueType().isInteger())
4202 Op.getOperand(1),
Op.getOperand(2));
4204 case Intrinsic::arm_neon_vabdu:
4206 Op.getOperand(1),
Op.getOperand(2));
4207 case Intrinsic::arm_neon_vmulls:
4208 case Intrinsic::arm_neon_vmullu: {
4209 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
4212 Op.getOperand(1),
Op.getOperand(2));
4214 case Intrinsic::arm_neon_vminnm:
4215 case Intrinsic::arm_neon_vmaxnm: {
4216 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
4219 Op.getOperand(1),
Op.getOperand(2));
4221 case Intrinsic::arm_neon_vminu:
4222 case Intrinsic::arm_neon_vmaxu: {
4223 if (
Op.getValueType().isFloatingPoint())
4225 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
4228 Op.getOperand(1),
Op.getOperand(2));
4230 case Intrinsic::arm_neon_vmins:
4231 case Intrinsic::arm_neon_vmaxs: {
4233 if (!
Op.getValueType().isFloatingPoint()) {
4234 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4237 Op.getOperand(1),
Op.getOperand(2));
4239 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4242 Op.getOperand(1),
Op.getOperand(2));
4244 case Intrinsic::arm_neon_vtbl1:
4246 Op.getOperand(1),
Op.getOperand(2));
4247 case Intrinsic::arm_neon_vtbl2:
4249 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4250 case Intrinsic::arm_mve_pred_i2v:
4251 case Intrinsic::arm_mve_pred_v2i:
4254 case Intrinsic::arm_mve_vreinterpretq:
4257 case Intrinsic::arm_mve_lsll:
4259 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4260 case Intrinsic::arm_mve_asrl:
4262 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4273 if (!Subtarget->hasDataBarrier()) {
4277 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4278 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4289 }
else if (Subtarget->preferISHSTBarriers() &&
4298 DAG.
getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4306 (!Subtarget->
isThumb1Only() && Subtarget->hasV5TEOps())))
4308 return Op.getOperand(0);
4311 unsigned isRead =
~Op.getConstantOperandVal(2) & 1;
4313 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4315 return Op.getOperand(0);
4317 unsigned isData =
Op.getConstantOperandVal(4);
4318 if (Subtarget->isThumb()) {
4320 isRead = ~isRead & 1;
4321 isData = ~isData & 1;
4338 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
4339 return DAG.
getStore(
Op.getOperand(0), dl, FR,
Op.getOperand(1),
4347 const SDLoc &dl)
const {
4353 RC = &ARM::tGPRRegClass;
4355 RC = &ARM::GPRRegClass;
4369 MVT::i32, dl, Root, FIN,
4390 const Value *OrigArg,
4391 unsigned InRegsParamRecordIdx,
4392 int ArgOffset,
unsigned ArgSize)
const {
4407 unsigned RBegin, REnd;
4417 ArgOffset = -4 * (ARM::R4 - RBegin);
4427 for (
unsigned Reg = RBegin, i = 0;
Reg < REnd; ++
Reg, ++i) {
4436 if (!MemOps.
empty())
4445 unsigned TotalArgRegsSaveSize,
4446 bool ForceMutable)
const {
4457 CCInfo.
getStackSize(), std::max(4U, TotalArgRegsSaveSize));
4461bool ARMTargetLowering::splitValueIntoRegisterParts(
4463 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
4465 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4477SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4479 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
4480 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4493SDValue ARMTargetLowering::LowerFormalArguments(
4509 unsigned CurArgIdx = 0;
4521 unsigned ArgRegBegin = ARM::R4;
4528 if (!
Flags.isByVal())
4532 unsigned RBegin, REnd;
4534 ArgRegBegin = std::min(ArgRegBegin, RBegin);
4540 int lastInsIndex = -1;
4544 ArgRegBegin = std::min(ArgRegBegin, (
unsigned)
GPRArgRegs[RegIdx]);
4547 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4551 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4553 if (Ins[VA.
getValNo()].isOrigArg()) {
4554 std::advance(CurOrigArg,
4555 Ins[VA.
getValNo()].getOrigArgIndex() - CurArgIdx);
4567 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4574 MVT::f64, dl, Chain, FIN,
4577 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4585 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4589 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4590 RC = &ARM::HPRRegClass;
4591 else if (RegVT == MVT::f32)
4592 RC = &ARM::SPRRegClass;
4593 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4594 RegVT == MVT::v4bf16)
4595 RC = &ARM::DPRRegClass;
4596 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4597 RegVT == MVT::v8bf16)
4598 RC = &ARM::QPRRegClass;
4599 else if (RegVT == MVT::i32)
4601 : &ARM::GPRRegClass;
4647 assert(VA.
getValVT() != MVT::i64 &&
"i64 should already be lowered");
4653 if (index != lastInsIndex)
4661 if (
Flags.isByVal()) {
4662 assert(Ins[index].isOrigArg() &&
4663 "Byval arguments cannot be implicit");
4667 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4682 lastInsIndex = index;
4689 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.
getStackSize(),
4690 TotalArgRegsSaveSize);
4694 "secure entry function must not be variadic", dl.
getDebugLoc());
4705 StackArgSize =
alignTo(StackArgSize,
DL.getStackAlignment());
4714 "secure entry function requires arguments on stack", dl.
getDebugLoc());
4724 return CFP->getValueAPF().isPosZero();
4728 SDValue WrapperOp =
Op.getOperand(1).getOperand(0);
4730 if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
4731 return CFP->getValueAPF().isPosZero();
4734 Op->getValueType(0) == MVT::f64) {
4749 const SDLoc &dl)
const {
4751 unsigned C = RHSC->getZExtValue();
4817 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(RHS) &&
4819 unsigned Mask =
LHS.getConstantOperandVal(1);
4820 auto *RHSC = cast<ConstantSDNode>(
RHS.getNode());
4821 uint64_t RHSV = RHSC->getZExtValue();
4822 if (
isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4824 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4839 isa<ConstantSDNode>(RHS) &&
RHS->getAsZExtVal() == 0x80000000U &&
4841 LHS.getConstantOperandVal(1) < 31) {
4842 unsigned ShiftAmt =
LHS.getConstantOperandVal(1) + 1;
4882 return DAG.
getNode(CompareType, dl, MVT::Glue, LHS, RHS);
4888 bool Signaling)
const {
4889 assert(Subtarget->hasFP64() ||
RHS.getValueType() != MVT::f64);
4893 dl, MVT::Glue, LHS, RHS);
4896 dl, MVT::Glue, LHS);
4904 unsigned Opc =
Cmp.getOpcode();
4907 return DAG.
getNode(Opc,
DL, MVT::Glue,
Cmp.getOperand(0),
Cmp.getOperand(1));
4911 Opc =
Cmp.getOpcode();
4925std::pair<SDValue, SDValue>
4928 assert(
Op.getValueType() == MVT::i32 &&
"Unsupported value type");
4940 switch (
Op.getOpcode()) {
4953 DAG.
getVTList(
Op.getValueType(), MVT::i32), LHS, RHS)
4992 return std::make_pair(
Value, OverflowCmp);
5003 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Op, DAG, ARMcc);
5009 EVT VT =
Op.getValueType();
5012 ARMcc, CCR, OverflowCmp);
5052 EVT VT =
Op.getValueType();
5056 switch (
Op.getOpcode()) {
5081 EVT VT =
Op.getValueType();
5082 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->
isThumb1Only())
5092 switch (
Op->getOpcode()) {
5108 switch (
Op->getOpcode()) {
5127 DAG.
getNode(NewOpcode, dl, MVT::i32,
5138 unsigned Opc =
Cond.getOpcode();
5140 if (
Cond.getResNo() == 1 &&
5148 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5150 EVT VT =
Op.getValueType();
5152 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
5163 dyn_cast<ConstantSDNode>(
Cond.getOperand(0));
5165 dyn_cast<ConstantSDNode>(
Cond.getOperand(1));
5167 if (CMOVTrue && CMOVFalse) {
5173 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5175 False = SelectFalse;
5176 }
else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5182 EVT VT =
Op.getValueType();
5187 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG);
5203 bool &swpCmpOps,
bool &swpVselOps) {
5231 swpCmpOps = !swpCmpOps;
5232 swpVselOps = !swpVselOps;
5255 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5257 DAG.
getVTList(MVT::i32, MVT::i32), FalseVal);
5259 DAG.
getVTList(MVT::i32, MVT::i32), TrueVal);
5269 ARMcc, CCR, duplicateCmp(Cmp, DAG));
5296 ((K ==
LHS && K == TrueVal) || (K ==
RHS && K == FalseVal))) ||
5298 ((K ==
RHS && K == TrueVal) || (K ==
LHS && K == FalseVal)));
5319 EVT VT =
Op.getValueType();
5326 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
5341 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5349 if (!isa<ConstantSDNode>(K1) || !isa<ConstantSDNode>(K2))
5352 int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue();
5353 int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue();
5354 int64_t PosVal = std::max(Val1, Val2);
5355 int64_t NegVal = std::min(Val1, Val2);
5401 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
5402 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5407 if (*K != KTmp || V != VTmp)
5418bool ARMTargetLowering::isUnsupportedFloatingType(
EVT VT)
const {
5422 return !Subtarget->hasFP64();
5424 return !Subtarget->hasFullFP16();
5429 EVT VT =
Op.getValueType();
5433 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->
isThumb2())
5445 if (VT == MVT::i32 &&
5465 if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
5466 LHS.getValueType() == MVT::i32 &&
RHS.getValueType() == MVT::i32) {
5469 unsigned Opcode = 0;
5471 if (TVal == ~FVal) {
5473 }
else if (TVal == ~FVal + 1) {
5475 }
else if (TVal + 1 == FVal) {
5477 }
else if (TVal == FVal + 1) {
5509 return DAG.
getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
5513 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5515 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5519 if (!
RHS.getNode()) {
5525 if (
LHS.getValueType() == MVT::i32) {
5537 TrueVal.getValueType() == MVT::f32 ||
5538 TrueVal.getValueType() == MVT::f64)) {
5553 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5565 (
TrueVal.getValueType() == MVT::f16 ||
5566 TrueVal.getValueType() == MVT::f32 ||
5567 TrueVal.getValueType() == MVT::f64)) {
5568 bool swpCmpOps =
false;
5569 bool swpVselOps =
false;
5584 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5588 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
5589 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
5599 if (!
N->hasOneUse())
5602 if (!
N->getNumValues())
5604 EVT VT =
Op.getValueType();
5605 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5622 return DAG.
getLoad(MVT::i32,
SDLoc(
Op), Ld->getChain(), Ld->getBasePtr(),
5623 Ld->getPointerInfo(), Ld->getAlign(),
5624 Ld->getMemOperand()->getFlags());
5642 DAG.
getLoad(MVT::i32, dl, Ld->getChain(),
Ptr, Ld->getPointerInfo(),
5643 Ld->getAlign(), Ld->getMemOperand()->getFlags());
5645 EVT PtrType =
Ptr.getValueType();
5648 RetVal2 = DAG.
getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5649 Ld->getPointerInfo().getWithOffset(4),
5651 Ld->getMemOperand()->getFlags());
5669 bool LHSSeenZero =
false;
5671 bool RHSSeenZero =
false;
5673 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5684 if (
LHS.getValueType() == MVT::f32) {
5692 Chain, Dest, ARMcc, CCR, Cmp);
5704 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5719 unsigned Opc =
Cond.getOpcode();
5722 if (
Cond.getResNo() == 1 &&
5732 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5756 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5758 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5762 if (!
RHS.getNode()) {
5770 unsigned Opc =
LHS.getOpcode();
5784 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
LHS.getValue(0), DAG, ARMcc);
5799 if (
LHS.getValueType() == MVT::i32) {
5804 Chain, Dest, ARMcc, CCR, Cmp);
5810 if (
SDValue Result = OptimizeVFPBrcond(
Op, DAG))
5821 SDValue Ops[] = { Chain, Dest, ARMcc, CCR,
Cmp };
5843 if (Subtarget->
isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5849 Addr,
Op.getOperand(2), JTI);
5855 Chain =
Addr.getValue(1);
5862 Chain =
Addr.getValue(1);
5868 EVT VT =
Op.getValueType();
5871 if (
Op.getValueType().getVectorElementType() == MVT::i32) {
5872 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
5880 const EVT OpTy =
Op.getOperand(0).getValueType();
5881 if (OpTy == MVT::v4f32)
5883 else if (OpTy == MVT::v4f16 && HasFullFP16)
5885 else if (OpTy == MVT::v8f16 && HasFullFP16)
5890 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5893 Op = DAG.
getNode(
Op.getOpcode(), dl, NewTy,
Op.getOperand(0));
5898 EVT VT =
Op.getValueType();
5902 bool IsStrict =
Op->isStrictFPOpcode();
5903 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
5905 if (isUnsupportedFloatingType(SrcVal.
getValueType())) {
5915 MakeLibCallOptions CallOptions;
5918 std::tie(Result, Chain) =
makeLibCall(DAG, LC,
Op.getValueType(), SrcVal,
5919 CallOptions, Loc, Chain);
5929 Loc,
Op.getValueType(), SrcVal);
5938 EVT VT =
Op.getValueType();
5939 EVT ToVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
5940 EVT FromVT =
Op.getOperand(0).getValueType();
5942 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5944 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5945 Subtarget->hasFP64())
5947 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5948 Subtarget->hasFullFP16())
5950 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5951 Subtarget->hasMVEFloatOps())
5953 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5954 Subtarget->hasMVEFloatOps())
5957 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5974 EVT VT =
Op.getValueType();
5977 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
5983 assert((
Op.getOperand(0).getValueType() == MVT::v4i16 ||
5984 Op.getOperand(0).getValueType() == MVT::v8i16) &&
5985 "Invalid type for custom lowering!");
5990 if (VT == MVT::v4f32)
5991 DestVecType = MVT::v4i32;
5992 else if (VT == MVT::v4f16 && HasFullFP16)
5993 DestVecType = MVT::v4i16;
5994 else if (VT == MVT::v8f16 && HasFullFP16)
5995 DestVecType = MVT::v8i16;
6001 switch (
Op.getOpcode()) {
6013 Op = DAG.
getNode(CastOpc, dl, DestVecType,
Op.getOperand(0));
6018 EVT VT =
Op.getValueType();
6021 if (isUnsupportedFloatingType(VT)) {
6029 MakeLibCallOptions CallOptions;
6031 CallOptions,
SDLoc(
Op)).first;
6042 EVT VT =
Op.getValueType();
6046 bool UseNEON = !InGPR && Subtarget->hasNEON();
6053 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
6060 if (SrcVT == MVT::f32) {
6066 }
else if (VT == MVT::f32)
6082 if (VT == MVT::f32) {
6094 if (SrcVT == MVT::f64)
6103 if (VT == MVT::f32) {
6127 EVT VT =
Op.getValueType();
6129 unsigned Depth =
Op.getConstantOperandVal(0);
6131 SDValue FrameAddr = LowerFRAMEADDR(
Op, DAG);
6150 EVT VT =
Op.getValueType();
6152 unsigned Depth =
Op.getConstantOperandVal(0);
6166 .
Case(
"sp", ARM::SP)
6181 assert(
N->getValueType(0) == MVT::i64
6182 &&
"ExpandREAD_REGISTER called for non-i64 type result.");
6185 DAG.
getVTList(MVT::i32, MVT::i32, MVT::Other),
6191 Results.push_back(Read.getOperand(0));
6225 const APInt &APIntIndex =
Index->getAPIntValue();
6227 NewIndex *= APIntIndex;
6257 EVT SrcVT =
Op.getValueType();
6258 EVT DstVT =
N->getValueType(0);
6260 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6261 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6265 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6266 (SrcVT == MVT::f16 || SrcVT == MVT::bf16))
6271 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6275 if (SrcVT == MVT::i64 && TLI.
isTypeLegal(DstVT)) {
6287 if (DstVT == MVT::i64 && TLI.
isTypeLegal(SrcVT)) {
6324 EVT VT =
Op.getValueType();
6343 SDValue LoBigShift = DAG.
getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6351 ? DAG.
getNode(Opc, dl, VT, ShOpHi,
6368 EVT VT =
Op.getValueType();
6411 DAG.
getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6463 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6491 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6521 EVT VT =
N->getValueType(0);
6522 if (VT.
isVector() && ST->hasNEON()) {
6531 if (ElemTy == MVT::i8) {
6539 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6554 if (ElemTy == MVT::i64) {
6567 if (!ST->hasV6T2Ops())
6576 EVT VT =
N->getValueType(0);
6579 assert(ST->hasNEON() &&
"Custom ctpop lowering requires NEON.");
6580 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6581 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6582 "Unexpected type for custom ctpop lowering");
6590 unsigned EltSize = 8;
6613 Op =
Op.getOperand(0);
6615 APInt SplatBits, SplatUndef;
6616 unsigned SplatBitSize;
6619 !BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6621 SplatBitSize > ElementBits)
6632 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6636 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6647 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6652 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6653 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6662 EVT VT =
N->getValueType(0);
6684 "unexpected vector shift opcode");
6686 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
6687 unsigned VShiftOpc =
6689 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
6695 EVT ShiftVT =
N->getOperand(1).getValueType();
6698 unsigned VShiftOpc =
6700 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0), NegatedCount);
6705 EVT VT =
N->getValueType(0);
6714 "Unknown shift to lower!");
6716 unsigned ShOpc =
N->getOpcode();
6717 if (ST->hasMVEIntegerOps()) {
6747 DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6761 if (ST->isThumb1Only())
6766 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6782 bool Invert =
false;
6789 EVT VT =
Op.getValueType();
6797 assert(ST->hasMVEIntegerOps() &&
6798 "No hardware support for integer vector comparison!");
6800 if (
Op.getValueType().getVectorElementType() != MVT::i1)
6825 Merged = DAG.
getNOT(dl, Merged, CmpVT);
6835 switch (SetCCOpcode) {
6839 if (ST->hasMVEFloatOps()) {
6842 Invert =
true; [[fallthrough]];
6847 case ISD::SETLT: Swap =
true; [[fallthrough]];
6851 case ISD::SETLE: Swap =
true; [[fallthrough]];
6867 Result = DAG.
getNOT(dl, Result, VT);
6870 case ISD::SETUO: Invert =
true; [[fallthrough]];
6879 Result = DAG.
getNOT(dl, Result, VT);
6885 switch (SetCCOpcode) {
6888 if (ST->hasMVEIntegerOps()) {
6891 Invert =
true; [[fallthrough]];
6894 case ISD::SETLT: Swap =
true; [[fallthrough]];
6896 case ISD::SETLE: Swap =
true; [[fallthrough]];
6905 if (ST->hasNEON() && Opc ==
ARMCC::EQ) {
6921 Result = DAG.
getNOT(dl, Result, VT);
6955 Result = DAG.
getNOT(dl, Result, VT);
6967 assert(
LHS.getSimpleValueType().isInteger() &&
"SETCCCARRY is integer only.");
6997 unsigned OpCmode, Imm;
7008 switch (SplatBitSize) {
7013 assert((SplatBits & ~0xff) == 0 &&
"one byte splat value is too big");
7016 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
7021 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
7022 if ((SplatBits & ~0xff) == 0) {
7028 if ((SplatBits & ~0xff00) == 0) {
7031 Imm = SplatBits >> 8;
7041 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
7042 if ((SplatBits & ~0xff) == 0) {
7048 if ((SplatBits & ~0xff00) == 0) {
7051 Imm = SplatBits >> 8;
7054 if ((SplatBits & ~0xff0000) == 0) {
7057 Imm = SplatBits >> 16;
7060 if ((SplatBits & ~0xff000000) == 0) {
7063 Imm = SplatBits >> 24;
7070 if ((SplatBits & ~0xffff) == 0 &&
7071 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
7074 Imm = SplatBits >> 8;
7082 if ((SplatBits & ~0xffffff) == 0 &&
7083 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
7086 Imm = SplatBits >> 16;
7102 unsigned ImmMask = 1;
7104 for (
int ByteNum = 0; ByteNum < 8; ++ByteNum) {
7105 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
7107 }
else if ((SplatBits & BitMask) != 0) {
7117 unsigned Mask = (1 << BytesPerElem) - 1;
7118 unsigned NumElems = 8 / BytesPerElem;
7119 unsigned NewImm = 0;
7120 for (
unsigned ElemNum = 0; ElemNum < NumElems; ++ElemNum) {
7121 unsigned Elem = ((Imm >> ElemNum * BytesPerElem) & Mask);
7122 NewImm |= Elem << (NumElems - ElemNum - 1) * BytesPerElem;
7129 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
7143 EVT VT =
Op.getValueType();
7144 bool IsDouble = (VT == MVT::f64);
7150 if (
ST->genExecuteOnly()) {
7152 assert((!
ST->isThumb1Only() ||
ST->hasV8MBaselineOps()) &&
7153 "Unexpected architecture");
7176 if (!
ST->hasVFP3Base())
7181 if (IsDouble && !Subtarget->hasFP64())
7188 if (IsDouble || !
ST->useNEONForSinglePrecisionFP()) {
7206 if (!
ST->hasNEON() || (!IsDouble && !
ST->useNEONForSinglePrecisionFP()))
7215 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7269 unsigned ExpectedElt = Imm;
7270 for (
unsigned i = 1; i < NumElts; ++i) {
7274 if (ExpectedElt == NumElts)
7277 if (M[i] < 0)
continue;
7278 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7286 bool &ReverseVEXT,
unsigned &Imm) {
7288 ReverseVEXT =
false;
7299 unsigned ExpectedElt = Imm;
7300 for (
unsigned i = 1; i < NumElts; ++i) {
7304 if (ExpectedElt == NumElts * 2) {
7309 if (M[i] < 0)
continue;
7310 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7325 return VT == MVT::v8i8 && M.size() == 8;
7330 if (Mask.size() == Elements * 2)
7331 return Index / Elements;
7332 return Mask[
Index] == 0 ? 0 : 1;
7362 if (M.size() != NumElts && M.size() != NumElts*2)
7370 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7372 for (
unsigned j = 0; j < NumElts; j += 2) {
7373 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7374 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + NumElts + WhichResult))
7379 if (M.size() == NumElts*2)
7394 if (M.size() != NumElts && M.size() != NumElts*2)
7397 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7399 for (
unsigned j = 0; j < NumElts; j += 2) {
7400 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7401 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + WhichResult))
7406 if (M.size() == NumElts*2)
7426 if (M.size() != NumElts && M.size() != NumElts*2)
7429 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7431 for (
unsigned j = 0; j < NumElts; ++j) {
7432 if (M[i+j] >= 0 && (
unsigned) M[i+j] != 2 * j + WhichResult)
7437 if (M.size() == NumElts*2)
7456 if (M.size() != NumElts && M.size() != NumElts*2)
7459 unsigned Half = NumElts / 2;
7460 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7462 for (
unsigned j = 0; j < NumElts; j += Half) {
7463 unsigned Idx = WhichResult;
7464 for (
unsigned k = 0; k < Half; ++k) {
7465 int MIdx = M[i + j + k];
7466 if (MIdx >= 0 && (
unsigned) MIdx !=
Idx)
7473 if (M.size() == NumElts*2)
7497 if (M.size() != NumElts && M.size() != NumElts*2)
7500 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7502 unsigned Idx = WhichResult * NumElts / 2;
7503 for (
unsigned j = 0; j < NumElts; j += 2) {
7504 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7505 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx + NumElts))
7511 if (M.size() == NumElts*2)
7530 if (M.size() != NumElts && M.size() != NumElts*2)
7533 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7535 unsigned Idx = WhichResult * NumElts / 2;
7536 for (
unsigned j = 0; j < NumElts; j += 2) {
7537 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7538 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx))
7544 if (M.size() == NumElts*2)
7557 unsigned &WhichResult,
7560 if (
isVTRNMask(ShuffleMask, VT, WhichResult))
7562 if (
isVUZPMask(ShuffleMask, VT, WhichResult))
7564 if (
isVZIPMask(ShuffleMask, VT, WhichResult))
7582 if (NumElts != M.size())
7586 for (
unsigned i = 0; i != NumElts; ++i)
7587 if (M[i] >= 0 && M[i] != (
int) (NumElts - 1 - i))
7596 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7604 int Ofs = Top ? 1 : 0;
7605 int Upper = SingleSource ? 0 : NumElts;
7606 for (
int i = 0, e = NumElts / 2; i != e; ++i) {
7607 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7609 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs +
Upper)
7618 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7627 unsigned Offset = Top ? 0 : 1;
7628 unsigned N = SingleSource ? 0 : NumElts;
7629 for (
unsigned i = 0; i < NumElts; i += 2) {
7630 if (M[i] >= 0 && M[i] != (
int)i)
7632 if (M[i + 1] >= 0 && M[i + 1] != (
int)(
N + i +
Offset))
7641 if (NumElts != M.size())
7649 unsigned Off0 = rev ? NumElts / 2 : 0;
7650 unsigned Off1 = rev ? 0 : NumElts / 2;
7651 for (
unsigned i = 0; i < NumElts; i += 2) {
7652 if (M[i] >= 0 && M[i] != (
int)(Off0 + i / 2))
7654 if (M[i + 1] >= 0 && M[i + 1] != (
int)(Off1 + i / 2))
7670 if (!ST->hasMVEFloatOps())
7675 if (VT != MVT::v8f16)
7696 for (
unsigned i = 1; i < 4; i++) {
7723 if (!ST->hasMVEFloatOps())
7728 if (VT != MVT::v4f32)
7744 for (
unsigned i = 1; i < 4; i++) {
7765 if (!isa<ConstantSDNode>(
N))
7767 Val =
N->getAsZExtVal();
7769 if (ST->isThumb1Only()) {
7770 if (Val <= 255 || ~Val <= 255)
7782 EVT VT =
Op.getValueType();
7784 assert(ST->hasMVEIntegerOps() &&
"LowerBUILD_VECTOR_i1 called without MVE!");
7788 unsigned BitsPerBool;
7792 }
else if (NumElts == 4) {
7795 }
else if (NumElts == 8) {
7798 }
else if (NumElts == 16) {
7807 if (!isa<ConstantSDNode>(FirstOp) &&
7809 return U.get().isUndef() || U.get() == FirstOp;
7817 unsigned Bits32 = 0;
7818 for (
unsigned i = 0; i < NumElts; ++i) {
7820 if (!isa<ConstantSDNode>(V) && !V.isUndef())
7822 bool BitSet = V.isUndef() ?
false : V->getAsZExtVal();
7824 Bits32 |= BoolMask << (i * BitsPerBool);
7830 for (
unsigned i = 0; i < NumElts; ++i) {
7832 if (isa<ConstantSDNode>(V) || V.isUndef())
7843 if (!ST->hasMVEIntegerOps())
7847 EVT VT =
Op.getValueType();
7857 if (
N != 1 &&
N != 2 &&
N != 4 &&
N != 8)
7861 for (
unsigned I = 2;
I < NumElts;
I++) {
7877 switch (
N->getOpcode()) {
7886 return N->getOperand(1).getNode() ==
Op;
7888 switch (
N->getConstantOperandVal(0)) {
7889 case Intrinsic::arm_mve_add_predicated:
7890 case Intrinsic::arm_mve_mul_predicated:
7891 case Intrinsic::arm_mve_qadd_predicated:
7892 case Intrinsic::arm_mve_vhadd:
7893 case Intrinsic::arm_mve_hadd_predicated:
7894 case Intrinsic::arm_mve_vqdmulh:
7895 case Intrinsic::arm_mve_qdmulh_predicated:
7896 case Intrinsic::arm_mve_vqrdmulh:
7897 case Intrinsic::arm_mve_qrdmulh_predicated:
7898 case Intrinsic::arm_mve_vqdmull:
7899 case Intrinsic::arm_mve_vqdmull_predicated:
7901 case Intrinsic::arm_mve_sub_predicated:
7902 case Intrinsic::arm_mve_qsub_predicated:
7903 case Intrinsic::arm_mve_vhsub:
7904 case Intrinsic::arm_mve_hsub_predicated:
7905 return N->getOperand(2).getNode() ==
Op;
7920 EVT VT =
Op.getValueType();
7928 APInt SplatBits, SplatUndef;
7929 unsigned SplatBitSize;
7931 if (BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7938 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7940 [BVN](
const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
7941 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7942 : SplatBitSize == 16 ? MVT::v8i16
7949 if ((
ST->hasNEON() && SplatBitSize <= 64) ||
7950 (
ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7955 SplatBitSize, DAG, dl, VmovVT, VT,
VMOVModImm);
7963 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7965 NegatedImm, SplatUndef.
getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
7973 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7983 if (
ST->hasMVEIntegerOps() &&
7984 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
7985 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7986 : SplatBitSize == 16 ? MVT::v8i16
8003 bool isOnlyLowElement =
true;
8004 bool usesOnlyOneValue =
true;
8005 bool hasDominantValue =
false;
8012 for (
unsigned i = 0; i < NumElts; ++i) {
8017 isOnlyLowElement =
false;
8018 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
8021 ValueCounts.
insert(std::make_pair(V, 0));
8022 unsigned &Count = ValueCounts[
V];
8025 if (++Count > (NumElts / 2)) {
8026 hasDominantValue =
true;
8030 if (ValueCounts.
size() != 1)
8031 usesOnlyOneValue =
false;
8032 if (!
Value.getNode() && !ValueCounts.
empty())
8035 if (ValueCounts.
empty())
8047 if (hasDominantValue && EltSize <= 32) {
8058 (constIndex = dyn_cast<ConstantSDNode>(
Value->getOperand(1)))) {
8063 if (VT !=
Value->getOperand(0).getValueType()) {
8076 if (!usesOnlyOneValue) {
8079 for (
unsigned I = 0;
I < NumElts; ++
I) {
8094 assert(FVT == MVT::f32 || FVT == MVT::f16);
8095 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
8096 for (
unsigned i = 0; i < NumElts; ++i)
8101 Val = LowerBUILD_VECTOR(Val, DAG, ST);
8105 if (usesOnlyOneValue) {
8132 if (
ST->hasNEON() && VT.
is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
8152 if (EltSize >= 32) {
8158 for (
unsigned i = 0; i < NumElts; ++i)
8172 for (
unsigned i = 0 ; i < NumElts; ++i) {
8191 EVT VT =
Op.getValueType();
8194 struct ShuffleSourceInfo {
8196 unsigned MinElt = std::numeric_limits<unsigned>::max();
8197 unsigned MaxElt = 0;
8207 int WindowScale = 1;
8209 ShuffleSourceInfo(
SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
8211 bool operator ==(
SDValue OtherVec) {
return Vec == OtherVec; }
8217 for (
unsigned i = 0; i < NumElts; ++i) {
8225 }
else if (!isa<ConstantSDNode>(
V.getOperand(1))) {
8232 SDValue SourceVec =
V.getOperand(0);
8234 if (Source == Sources.
end())
8238 unsigned EltNo =
V.getConstantOperandVal(1);
8245 if (Sources.
size() > 2)
8251 for (
auto &Source : Sources) {
8252 EVT SrcEltTy =
Source.Vec.getValueType().getVectorElementType();
8253 if (SrcEltTy.
bitsLT(SmallestEltTy))
8254 SmallestEltTy = SrcEltTy;
8256 unsigned ResMultiplier =
8264 for (
auto &Src : Sources) {
8265 EVT SrcVT = Src.ShuffleVec.getValueType();
8269 if (SrcVTSize == VTSize)
8278 if (SrcVTSize < VTSize) {
8279 if (2 * SrcVTSize != VTSize)
8285 DAG.
getUNDEF(Src.ShuffleVec.getValueType()));
8289 if (SrcVTSize != 2 * VTSize)
8292 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8297 if (Src.MinElt >= NumSrcElts) {
8302 Src.WindowBase = -NumSrcElts;
8303 }
else if (Src.MaxElt < NumSrcElts) {
8320 Src.WindowBase = -Src.MinElt;
8327 for (
auto &Src : Sources) {
8328 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8329 if (SrcEltTy == SmallestEltTy)
8334 Src.WindowBase *= Src.WindowScale;
8340 assert(Src.ShuffleVec.getValueType() == ShuffleVT););
8347 if (
Entry.isUndef())
8351 int EltNo = cast<ConstantSDNode>(
Entry.getOperand(1))->getSExtValue();
8356 EVT OrigEltTy =
Entry.getOperand(0).getValueType().getVectorElementType();
8359 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8363 int *LaneMask = &
Mask[i * ResMultiplier];
8365 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8366 ExtractBase += NumElts * (Src - Sources.begin());
8367 for (
int j = 0;
j < LanesDefined; ++
j)
8368 LaneMask[j] = ExtractBase + j;
8374 assert(Sources.size() <= 2 &&
"Too many sources!");
8377 for (
unsigned i = 0; i < Sources.size(); ++i)
8406 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8426 unsigned PFIndexes[4];
8427 for (
unsigned i = 0; i != 4; ++i) {
8431 PFIndexes[i] = M[i];
8435 unsigned PFTableIndex =
8436 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8438 unsigned Cost = (PFEntry >> 30);
8444 bool ReverseVEXT, isV_UNDEF;
8445 unsigned Imm, WhichResult;
8448 if (EltSize >= 32 ||
8455 else if (Subtarget->hasNEON() &&
8460 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8463 else if (Subtarget->hasMVEIntegerOps() &&
8467 else if (Subtarget->hasMVEIntegerOps() &&
8481 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8482 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8483 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8486 if (LHSID == (1*9+2)*9+3)
return LHS;
8487 assert(LHSID == ((4*9+5)*9+6)*9+7 &&
"Illegal OP_COPY!");
8544 for (
int I : ShuffleMask)
8547 if (V2.getNode()->isUndef())
8557 EVT VT =
Op.getValueType();
8559 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8560 "Expect an v8i16/v16i8 type");
8566 std::vector<int> NewMask;
8570 NewMask.push_back(i);
8610 if (VT != MVT::v16i1)
8626 EVT VT =
Op.getValueType();
8630 assert(ST->hasMVEIntegerOps() &&
8631 "No support for vector shuffle of boolean predicates");
8657 "Expected identical vector type in expanded i1 shuffle!");
8661 PredAsVector2, ShuffleMask);
8666 if (VT == MVT::v2i1) {
8683 EVT VT =
Op.getValueType();
8687 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8688 "Unexpected vector type");
8690 int QuarterSize = NumElts / 4;
8699 for (
int i = 0; i <
Length; i++) {
8700 if (ShuffleMask[Start + i] >= 0) {
8701 if (ShuffleMask[Start + i] %
Length != i)
8703 MovIdx = ShuffleMask[Start + i] /
Length;
8711 for (
int i = 1; i <
Length; i++) {
8712 if (ShuffleMask[Start + i] >= 0 &&
8713 (ShuffleMask[Start + i] /
Length != MovIdx ||
8714 ShuffleMask[Start + i] %
Length != i))
8720 for (
int Part = 0; Part < 4; ++Part) {
8722 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8726 Input =
Op->getOperand(1);
8736 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8741 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8743 for (
int Part = 0; Part < 4; ++Part)
8744 for (
int i = 0; i < QuarterSize; i++)
8746 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8748 VT, dl,
Op->getOperand(0),
Op->getOperand(1), NewShuffleMask);
8751 for (
int Part = 0; Part < 4; ++Part)
8767 EVT VT =
Op.getValueType();
8779 for (
int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8783 if (Mask[i] != i + BaseOffset) {
8784 if (OffElement == -1)
8790 return NonUndef > 2 && OffElement != -1;
8794 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8796 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8807 ShuffleMask[OffElement] < (
int)NumElts ? V1 : V2,
8818 EVT VT =
Op.getValueType();
8822 if (ST->hasMVEIntegerOps() && EltSize == 1)
8833 if (EltSize <= 32) {
8837 if (Lane == -1) Lane = 0;
8848 bool IsScalarToVector =
true;
8851 IsScalarToVector =
false;
8854 if (IsScalarToVector)
8861 bool ReverseVEXT =
false;
8863 if (ST->hasNEON() &&
isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
8887 unsigned WhichResult = 0;
8888 bool isV_UNDEF =
false;
8889 if (ST->hasNEON()) {
8891 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8898 if (ST->hasMVEIntegerOps()) {
8933 }) &&
"Unexpected shuffle index into UNDEF operand!");
8936 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
8939 assert((WhichResult == 0) &&
8940 "In-place shuffle of concat can only have one result!");
8949 if (ST->hasMVEIntegerOps() && EltSize <= 32) {
8953 for (
bool Top : {
false,
true}) {
8954 for (
bool SingleSource : {
false,
true}) {
8955 if (
isTruncMask(ShuffleMask, VT, Top, SingleSource)) {
8960 SingleSource ? V1 : V2);
8976 unsigned PFIndexes[4];
8977 for (
unsigned i = 0; i != 4; ++i) {
8978 if (ShuffleMask[i] < 0)
8981 PFIndexes[i] = ShuffleMask[i];
8985 unsigned PFTableIndex =
8986 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8988 unsigned Cost = (PFEntry >> 30);
8994 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8995 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
9005 if (EltSize >= 32) {
9013 for (
unsigned i = 0; i < NumElts; ++i) {
9014 if (ShuffleMask[i] < 0)
9018 ShuffleMask[i] < (
int)NumElts ? V1 : V2,
9026 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
9030 if (ST->hasNEON() && VT == MVT::v8i8)
9034 if (ST->hasMVEIntegerOps())
9043 EVT VecVT =
Op.getOperand(0).getValueType();
9046 assert(ST->hasMVEIntegerOps() &&
9047 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9051 unsigned Lane =
Op.getConstantOperandVal(2);
9052 unsigned LaneWidth =
9054 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
9066 if (!isa<ConstantSDNode>(Lane))
9072 if (Subtarget->hasMVEIntegerOps() &&
9073 Op.getValueType().getScalarSizeInBits() == 1)
9097 IVecIn, IElt, Lane);
9106 EVT VecVT =
Op.getOperand(0).getValueType();
9109 assert(ST->hasMVEIntegerOps() &&
9110 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9114 unsigned Lane =
Op.getConstantOperandVal(1);
9115 unsigned LaneWidth =
9126 if (!isa<ConstantSDNode>(Lane))
9146 assert(
Op.getValueType().getScalarSizeInBits() == 1 &&
9147 "Unexpected custom CONCAT_VECTORS lowering");
9149 "Unexpected custom CONCAT_VECTORS lowering");
9150 assert(ST->hasMVEIntegerOps() &&
9151 "CONCAT_VECTORS lowering only supported for MVE");
9155 EVT Op2VT = V2.getValueType();
9156 assert(Op1VT == Op2VT &&
"Operand types don't match!");
9157 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
9158 "Unexpected i1 concat operations!");
9171 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
9185 auto ExtractInto = [&DAG, &dl](
SDValue NewV,
SDValue ConVec,
unsigned &j) {
9186 EVT NewVT = NewV.getValueType();
9187 EVT ConcatVT = ConVec.getValueType();
9188 unsigned ExtScale = 1;
9189 if (NewVT == MVT::v2f64) {
9203 ConVec = ExtractInto(NewV1, ConVec, j);
9204 ConVec = ExtractInto(NewV2, ConVec, j);
9214 while (ConcatOps.
size() > 1) {
9215 for (
unsigned I = 0, E = ConcatOps.
size();
I != E;
I += 2) {
9218 ConcatOps[
I / 2] = ConcatPair(V1, V2);
9222 return ConcatOps[0];
9227 EVT VT =
Op->getValueType(0);
9234 "unexpected CONCAT_VECTORS");
9255 EVT VT =
Op.getValueType();
9258 unsigned Index = V2->getAsZExtVal();
9261 "Unexpected custom EXTRACT_SUBVECTOR lowering");
9262 assert(ST->hasMVEIntegerOps() &&
9263 "EXTRACT_SUBVECTOR lowering only supported for MVE");
9273 EVT SubVT = MVT::v4i32;
9275 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j += 2) {
9290 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j++) {
9306 assert(ST->hasMVEIntegerOps() &&
"Expected MVE!");
9307 EVT VT =
N->getValueType(0);
9308 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9309 "Expected a vector i1 type!");
9311 EVT FromVT =
Op.getValueType();
9322 if (!Subtarget->hasMVEIntegerOps())
9325 EVT ToVT =
N->getValueType(0);
9368 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9370 EVT FromVT =
N->getOperand(0).getValueType();
9371 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9382 if (!Subtarget->hasMVEIntegerOps())
9387 EVT ToVT =
N->getValueType(0);
9388 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9391 EVT FromVT =
Op.getValueType();
9392 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9403 SDValue Ext1 = Ext.getValue(1);
9406 Ext = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext);
9407 Ext1 = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext1);
9419 EVT VT =
N->getValueType(0);
9421 SDNode *BVN =
N->getOperand(0).getNode();
9426 unsigned HiElt = 1 - LoElt;
9431 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9447 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
9448 SDNode *Elt =
N->getOperand(i).getNode();
9451 unsigned HalfSize = EltSize / 2;
9453 if (!
isIntN(HalfSize,
C->getSExtValue()))
9456 if (!
isUIntN(HalfSize,
C->getZExtValue()))
9495 switch (OrigSimpleTy) {
9511 unsigned ExtOpcode) {
9534 if (ExtendedTy == LD->getMemoryVT())
9535 return DAG.
getLoad(LD->getMemoryVT(),
SDLoc(LD), LD->getChain(),
9536 LD->getBasePtr(), LD->getPointerInfo(), LD->getAlign(),
9537 LD->getMemOperand()->getFlags());
9543 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
9544 LD->getMemoryVT(), LD->getAlign(),
9545 LD->getMemOperand()->getFlags());
9558 N->getOperand(0)->getValueType(0),
9564 "Expected extending load");
9570 DAG.
getNode(Opcode,
SDLoc(newLoad), LD->getValueType(0), newLoad);
9579 SDNode *BVN =
N->getOperand(0).getNode();
9581 BVN->
getValueType(0) == MVT::v4i32 &&
"expected v4i32 BUILD_VECTOR");
9589 EVT VT =
N->getValueType(0);
9595 for (
unsigned i = 0; i != NumElts; ++i) {
9596 const APInt &CInt =
N->getConstantOperandAPInt(i);
9605 unsigned Opcode =
N->getOpcode();
9607 SDNode *N0 =
N->getOperand(0).getNode();
9608 SDNode *N1 =
N->getOperand(1).getNode();
9616 unsigned Opcode =
N->getOpcode();
9618 SDNode *N0 =
N->getOperand(0).getNode();
9619 SDNode *N1 =
N->getOperand(1).getNode();
9629 EVT VT =
Op.getValueType();
9631 "unexpected type for custom-lowering ISD::MUL");
9632 SDNode *N0 =
Op.getOperand(0).getNode();
9633 SDNode *N1 =
Op.getOperand(1).getNode();
9634 unsigned NewOpc = 0;
9638 if (isN0SExt && isN1SExt)
9643 if (isN0ZExt && isN1ZExt)
9645 else if (isN1SExt || isN1ZExt) {
9662 if (VT == MVT::v2i64)
9679 "unexpected types for extended operands to VMULL");
9680 return DAG.
getNode(NewOpc,
DL, VT, Op0, Op1);
9715 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9749 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9752 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9773 EVT VT =
Op.getValueType();
9774 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9775 "unexpected type for custom-lowering ISD::SDIV");
9782 if (VT == MVT::v8i8) {
9810 EVT VT =
Op.getValueType();
9811 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9812 "unexpected type for custom-lowering ISD::UDIV");
9819 if (VT == MVT::v8i8) {
9858 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9861 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9865 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9886 EVT VT =
N->getValueType(0);
9900 Op.getOperand(1), Carry);
9914 Op.getOperand(1), Carry);
9947 bool ShouldUseSRet = Subtarget->
isAPCS_ABI();
9949 if (ShouldUseSRet) {
9959 Entry.IsSExt =
false;
9960 Entry.IsZExt =
false;
9961 Entry.IsSRet =
true;
9962 Args.push_back(Entry);
9969 Entry.IsSExt =
false;
9970 Entry.IsZExt =
false;
9971 Args.push_back(Entry);
9974 (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
9984 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
9987 return CallResult.first;
10006 EVT VT =
Op.getValueType();
10007 assert((VT == MVT::i32 || VT == MVT::i64) &&
10008 "unexpected type for custom lowering DIV");
10014 const char *
Name =
nullptr;
10016 Name = (VT == MVT::i32) ?
"__rt_sdiv" :
"__rt_sdiv64";
10018 Name = (VT == MVT::i32) ?
"__rt_udiv" :
"__rt_udiv64";
10024 for (
auto AI : {1, 0}) {
10026 Arg.Node =
Op.getOperand(AI);
10027 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.
getContext());
10028 Args.push_back(Arg);
10031 CallLoweringInfo CLI(DAG);
10035 ES, std::move(Args));
10045ARMTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
10053 const bool MinSize =
ST.hasMinSize();
10054 const bool HasDivide =
ST.isThumb() ?
ST.hasDivideInThumbMode()
10055 :
ST.hasDivideInARMMode();
10059 if (
N->getOperand(0).getValueType().isVector())
10064 if (!(MinSize && HasDivide))
10077 if (Divisor.
sgt(128))
10085 assert(
Op.getValueType() == MVT::i32 &&
10086 "unexpected type for custom lowering DIV");
10092 return LowerWindowsDIVLibCall(
Op, DAG,
Signed, DBZCHK);
10098 if (
N->getValueType(0) == MVT::i32)
10106void ARMTargetLowering::ExpandDIV_Windows(
10112 assert(
Op.getValueType() == MVT::i64 &&
10113 "unexpected type for custom lowering DIV");
10130 EVT MemVT = LD->getMemoryVT();
10131 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10132 MemVT == MVT::v16i1) &&
10133 "Expected a predicate type!");
10134 assert(MemVT ==
Op.getValueType());
10136 "Expected a non-extending load");
10137 assert(LD->isUnindexed() &&
"Expected a unindexed load");
10151 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(),
10153 LD->getMemOperand());
10160 if (MemVT != MVT::v16i1)
10169 EVT MemVT =
LD->getMemoryVT();
10170 assert(
LD->isUnindexed() &&
"Loads should be unindexed at this point.");
10172 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10178 {LD->getChain(), LD->getBasePtr()}, MemVT,
LD->getMemOperand());
10188 EVT MemVT = ST->getMemoryVT();
10189 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10190 MemVT == MVT::v16i1) &&
10191 "Expected a predicate type!");
10192 assert(MemVT == ST->getValue().getValueType());
10193 assert(!ST->isTruncatingStore() &&
"Expected a non-extending store");
10194 assert(ST->isUnindexed() &&
"Expected a unindexed store");
10199 SDValue Build = ST->getValue();
10200 if (MemVT != MVT::v16i1) {
10219 ST->getChain(), dl, GRP, ST->getBasePtr(),
10221 ST->getMemOperand());
10227 EVT MemVT = ST->getMemoryVT();
10228 assert(ST->isUnindexed() &&
"Stores should be unindexed at this point.");
10230 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10246 {ST->getChain(), Lo, Hi, ST->getBasePtr()},
10247 MemVT, ST->getMemOperand());
10248 }
else if (Subtarget->hasMVEIntegerOps() &&
10249 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10250 MemVT == MVT::v16i1))) {
10265 MVT VT =
Op.getSimpleValueType();
10267 SDValue PassThru =
N->getPassThru();
10278 VT, dl,
N->getChain(),
N->getBasePtr(),
N->getOffset(), Mask, ZeroVec,
10279 N->getMemoryVT(),
N->getMemOperand(),
N->getAddressingMode(),
10280 N->getExtensionType(),
N->isExpandingLoad());
10285 if (!PassThru.
isUndef() && !PassThruIsCastZero)
10292 if (!ST->hasMVEIntegerOps())
10296 unsigned BaseOpcode = 0;
10297 switch (
Op->getOpcode()) {
10313 unsigned NumActiveLanes = NumElts;
10315 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10316 NumActiveLanes == 2) &&
10317 "Only expected a power 2 vector size");
10321 while (NumActiveLanes > 4) {
10324 Op0 = DAG.
getNode(BaseOpcode, dl, VT, Op0, Rev);
10325 NumActiveLanes /= 2;
10329 if (NumActiveLanes == 4) {
10339 SDValue Res0 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10340 SDValue Res1 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext2, Ext3,
Op->getFlags());
10341 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res0, Res1,
Op->getFlags());
10347 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10351 if (EltVT !=
Op->getValueType(0))
10358 if (!ST->hasMVEFloatOps())
10365 if (!ST->hasNEON())
10373 unsigned PairwiseIntrinsic = 0;
10374 switch (
Op->getOpcode()) {
10378 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
10381 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
10384 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
10387 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10393 unsigned NumActiveLanes = NumElts;
10395 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10396 NumActiveLanes == 2) &&
10397 "Only expected a power 2 vector size");
10403 VT =
Lo.getValueType();
10405 NumActiveLanes /= 2;
10409 while (NumActiveLanes > 1) {
10411 NumActiveLanes /= 2;
10418 if (EltVT !=
Op.getValueType()) {
10419 unsigned Extend = 0;
10420 switch (
Op->getOpcode()) {
10432 Res = DAG.
getNode(Extend, dl,
Op.getValueType(), Res);
10454 SDValue Ops[] = {
N->getOperand(0),
10464 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
10471 SDLoc dl(V.getNode());
10472 auto [VLo, VHi] = DAG.
SplitScalar(V, dl, MVT::i32, MVT::i32);
10480 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
10482 DAG.
getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
10488 assert(
N->getValueType(0) == MVT::i64 &&
10489 "AtomicCmpSwap on types less than 64 should be legal");
10490 SDValue Ops[] = {
N->getOperand(1),
10495 ARM::CMP_SWAP_64,
SDLoc(
N),
10496 DAG.
getVTList(MVT::Untyped, MVT::i32, MVT::Other), Ops);
10515 EVT VT =
Op.getValueType();
10524 if (isUnsupportedFloatingType(
LHS.getValueType())) {
10526 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS, Chain, IsSignaling);
10527 if (!
RHS.getNode()) {
10548 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10549 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, CCR, Cmp, DAG);
10551 ARMcc = DAG.
getConstant(CondCode2, dl, MVT::i32);
10552 Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10553 Result = getCMOV(dl, VT, Result, True, ARMcc, CCR, Cmp, DAG);
10569 switch (
Op.getOpcode()) {
10601 case ISD::BITCAST:
return ExpandBITCAST(
Op.getNode(), DAG, Subtarget);
10605 case ISD::SREM:
return LowerREM(
Op.getNode(), DAG);
10606 case ISD::UREM:
return LowerREM(
Op.getNode(), DAG);
10628 return LowerSET_FPMODE(
Op, DAG);
10630 return LowerRESET_FPMODE(
Op, DAG);
10634 return LowerDIV_Windows(
Op, DAG,
true);
10638 return LowerDIV_Windows(
Op, DAG,
false);
10645 return LowerSignedALUO(
Op, DAG);
10648 return LowerUnsignedALUO(
Op, DAG);
10682 return LowerDYNAMIC_STACKALLOC(
Op, DAG);
10691 return LowerSPONENTRY(
Op, DAG);
10698 unsigned IntNo =
N->getConstantOperandVal(0);
10700 if (IntNo == Intrinsic::arm_smlald)
10702 else if (IntNo == Intrinsic::arm_smlaldx)
10704 else if (IntNo == Intrinsic::arm_smlsld)
10706 else if (IntNo == Intrinsic::arm_smlsldx)
10713 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(3), dl, MVT::i32, MVT::i32);
10717 N->getOperand(1),
N->getOperand(2),
10729 switch (
N->getOpcode()) {
10736 Res = ExpandBITCAST(
N, DAG, Subtarget);
10745 Res = LowerREM(
N, DAG);
10749 Res = LowerDivRem(
SDValue(
N, 0), DAG);
10803 "ROPI/RWPI not currently supported with SjLj");
10812 bool isThumb = Subtarget->isThumb();
10813 bool isThumb2 = Subtarget->
isThumb2();
10816 unsigned PCAdj = (
isThumb || isThumb2) ? 4 : 8;
10822 : &ARM::GPRRegClass;
10840 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10846 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10852 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10870 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10875 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10880 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10885 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
10891 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
10906 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10912 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10936 : &ARM::GPRnopcRegClass;
10941 unsigned MaxCSNum = 0;
10949 if (!
II.isEHLabel())
10953 if (!MF->hasCallSiteLandingPad(
Sym))
continue;
10956 for (
unsigned Idx : CallSiteIdxs) {
10957 CallSiteNumToLPad[
Idx].push_back(&BB);
10958 MaxCSNum = std::max(MaxCSNum,
Idx);
10965 std::vector<MachineBasicBlock*> LPadList;
10967 LPadList.reserve(CallSiteNumToLPad.
size());
10968 for (
unsigned I = 1;
I <= MaxCSNum; ++
I) {
10971 LPadList.push_back(
MBB);
10976 assert(!LPadList.empty() &&
10977 "No landing pad destinations for the dispatch jump table!");
10991 unsigned trap_opcode;
10992 if (Subtarget->isThumb())
10993 trap_opcode = ARM::tTRAP;
10995 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP;
11004 MF->insert(MF->end(), DispatchBB);
11005 MF->insert(MF->end(), DispContBB);
11006 MF->insert(MF->end(), TrapBB);
11010 SetupEntryBlockForSjLj(
MI,
MBB, DispatchBB, FI);
11017 MIB =
BuildMI(DispatchBB, dl,
TII->get(ARM::Int_eh_sjlj_dispatchsetup));
11029 unsigned NumLPads = LPadList.size();
11031 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11032 BuildMI(DispatchBB, dl,
TII->get(ARM::t2LDRi12), NewVReg1)
11038 if (NumLPads < 256) {
11039 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPri))
11041 .
addImm(LPadList.size())
11044 Register VReg1 =
MRI->createVirtualRegister(TRC);
11045 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVi16), VReg1)
11046 .
addImm(NumLPads & 0xFFFF)
11049 unsigned VReg2 = VReg1;
11050 if ((NumLPads & 0xFFFF0000) != 0) {
11051 VReg2 =
MRI->createVirtualRegister(TRC);
11052 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVTi16), VReg2)
11058 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPrr))
11064 BuildMI(DispatchBB, dl,
TII->get(ARM::t2Bcc))
11069 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11070 BuildMI(DispContBB, dl,
TII->get(ARM::t2LEApcrelJT), NewVReg3)
11074 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11075 BuildMI(DispContBB, dl,
TII->get(ARM::t2ADDrs), NewVReg4)
11082 BuildMI(DispContBB, dl,
TII->get(ARM::t2BR_JT))
11086 }
else if (Subtarget->isThumb()) {
11087 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11088 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRspi), NewVReg1)
11094 if (NumLPads < 256) {
11095 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPi8))
11102 const Constant *
C = ConstantInt::get(Int32Ty, NumLPads);
11105 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
11108 Register VReg1 =
MRI->createVirtualRegister(TRC);
11109 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRpci))
11113 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPr))
11119 BuildMI(DispatchBB, dl,
TII->get(ARM::tBcc))
11124 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
11125 BuildMI(DispContBB, dl,
TII->get(ARM::tLSLri), NewVReg2)
11131 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11132 BuildMI(DispContBB, dl,
TII->get(ARM::tLEApcrelJT), NewVReg3)
11136 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11137 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg4)
11147 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11148 BuildMI(DispContBB, dl,
TII->get(ARM::tLDRi), NewVReg5)
11154 unsigned NewVReg6 = NewVReg5;
11155 if (IsPositionIndependent) {
11156 NewVReg6 =
MRI->createVirtualRegister(TRC);
11157 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg6)
11164 BuildMI(DispContBB, dl,
TII->get(ARM::tBR_JTr))
11168 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11169 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRi12), NewVReg1)
11175 if (NumLPads < 256) {
11176 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPri))
11180 }
else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) {
11181 Register VReg1 =
MRI->createVirtualRegister(TRC);
11182 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVi16), VReg1)
11183 .
addImm(NumLPads & 0xFFFF)
11186 unsigned VReg2 = VReg1;
11187 if ((NumLPads & 0xFFFF0000) != 0) {
11188 VReg2 =
MRI->createVirtualRegister(TRC);
11189 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVTi16), VReg2)
11195 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11202 const Constant *
C = ConstantInt::get(Int32Ty, NumLPads);
11205 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
11208 Register VReg1 =
MRI->createVirtualRegister(TRC);
11209 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRcp))
11214 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11225 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11226 BuildMI(DispContBB, dl,
TII->get(ARM::MOVsi), NewVReg3)
11231 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11232 BuildMI(DispContBB, dl,
TII->get(ARM::LEApcrelJT), NewVReg4)
11239 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11240 BuildMI(DispContBB, dl,
TII->get(ARM::LDRrs), NewVReg5)
11247 if (IsPositionIndependent) {
11248 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTadd))
11253 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTr))
11262 if (SeenMBBs.
insert(CurMBB).second)
11274 while (!Successors.empty()) {
11283 BB->normalizeSuccProbs();
11290 II = BB->rbegin(), IE = BB->rend();
II != IE; ++
II) {
11291 if (!
II->isCall())
continue;
11295 OI =
II->operands_begin(), OE =
II->operands_end();
11297 if (!OI->isReg())
continue;
11298 DefRegs[OI->getReg()] =
true;
11303 for (
unsigned i = 0; SavedRegs[i] != 0; ++i) {
11304 unsigned Reg = SavedRegs[i];
11306 !ARM::tGPRRegClass.contains(Reg) &&
11307 !ARM::hGPRRegClass.contains(Reg))
11309 if (Subtarget->
isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11311 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11324 MBBLPad->setIsEHPad(
false);
11327 MI.eraseFromParent();
11340static unsigned getLdOpcode(
unsigned LdSize,
bool IsThumb1,
bool IsThumb2) {
11342 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11343 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11345 return LdSize == 4 ? ARM::tLDRi
11346 : LdSize == 2 ? ARM::tLDRHi
11347 : LdSize == 1 ? ARM::tLDRBi : 0;
11349 return LdSize == 4 ? ARM::t2LDR_POST
11350 : LdSize == 2 ? ARM::t2LDRH_POST
11351 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11352 return LdSize == 4 ? ARM::LDR_POST_IMM
11353 : LdSize == 2 ? ARM::LDRH_POST
11354 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11359static unsigned getStOpcode(
unsigned StSize,
bool IsThumb1,
bool IsThumb2) {
11361 return StSize == 16 ? ARM::VST1q32wb_fixed
11362 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11364 return StSize == 4 ? ARM::tSTRi
11365 : StSize == 2 ? ARM::tSTRHi
11366 : StSize == 1 ? ARM::tSTRBi : 0;
11368 return StSize == 4 ? ARM::t2STR_POST
11369 : StSize == 2 ? ARM::t2STRH_POST
11370 : StSize == 1 ? ARM::t2STRB_POST : 0;
11371 return StSize == 4 ? ARM::STR_POST_IMM
11372 : StSize == 2 ? ARM::STRH_POST
11373 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11380 unsigned LdSize,
unsigned Data,
unsigned AddrIn,
11381 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11382 unsigned LdOpc =
getLdOpcode(LdSize, IsThumb1, IsThumb2);
11383 assert(LdOpc != 0 &&
"Should have a load opcode");
11390 }
else if (IsThumb1) {
11396 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11401 }
else if (IsThumb2) {
11421 unsigned StSize,
unsigned Data,
unsigned AddrIn,
11422 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11423 unsigned StOpc =
getStOpcode(StSize, IsThumb1, IsThumb2);
11424 assert(StOpc != 0 &&
"Should have a store opcode");
11426 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11431 }
else if (IsThumb1) {
11438 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11443 }
else if (IsThumb2) {
11444 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11450 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11471 unsigned SizeVal =
MI.getOperand(2).getImm();
11472 unsigned Alignment =
MI.getOperand(3).getImm();
11477 unsigned UnitSize = 0;
11482 bool IsThumb2 = Subtarget->
isThumb2();
11483 bool IsThumb = Subtarget->isThumb();
11485 if (Alignment & 1) {
11487 }
else if (Alignment & 2) {
11492 Subtarget->hasNEON()) {
11493 if ((Alignment % 16 == 0) && SizeVal >= 16)
11495 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11504 bool IsNeon = UnitSize >= 8;
11505 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11507 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11508 : UnitSize == 8 ? &ARM::DPRRegClass
11511 unsigned BytesLeft = SizeVal % UnitSize;
11512 unsigned LoopSize = SizeVal - BytesLeft;
11514 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11518 unsigned srcIn = src;
11519 unsigned destIn = dest;
11520 for (
unsigned i = 0; i < LoopSize; i+=UnitSize) {
11521 Register srcOut =
MRI.createVirtualRegister(TRC);
11522 Register destOut =
MRI.createVirtualRegister(TRC);
11523 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11525 IsThumb1, IsThumb2);
11527 IsThumb1, IsThumb2);
11535 for (
unsigned i = 0; i < BytesLeft; i++) {
11536 Register srcOut =
MRI.createVirtualRegister(TRC);
11537 Register destOut =
MRI.createVirtualRegister(TRC);
11538 Register scratch =
MRI.createVirtualRegister(TRC);
11540 IsThumb1, IsThumb2);
11542 IsThumb1, IsThumb2);
11546 MI.eraseFromParent();
11572 MF->
insert(It, loopMBB);
11573 MF->
insert(It, exitMBB);
11576 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
11586 Register varEnd =
MRI.createVirtualRegister(TRC);
11588 BuildMI(BB, dl,
TII->get(IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11591 }
else if (Subtarget->genExecuteOnly()) {
11592 assert(IsThumb &&
"Non-thumb expected to have used movt");
11597 const Constant *
C = ConstantInt::get(Int32Ty, LoopSize);
11628 Register varLoop =
MRI.createVirtualRegister(TRC);
11629 Register varPhi =
MRI.createVirtualRegister(TRC);
11630 Register srcLoop =
MRI.createVirtualRegister(TRC);
11631 Register srcPhi =
MRI.createVirtualRegister(TRC);
11632 Register destLoop =
MRI.createVirtualRegister(TRC);
11633 Register destPhi =
MRI.createVirtualRegister(TRC);
11641 BuildMI(BB, dl,
TII->get(ARM::PHI), destPhi)
11647 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11649 IsThumb1, IsThumb2);
11651 IsThumb1, IsThumb2);
11655 BuildMI(*BB, BB->
end(), dl,
TII->get(ARM::tSUBi8), varLoop)
11663 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
11672 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11681 auto StartOfExit = exitMBB->
begin();
11685 unsigned srcIn = srcLoop;
11686 unsigned destIn = destLoop;
11687 for (
unsigned i = 0; i < BytesLeft; i++) {
11688 Register srcOut =
MRI.createVirtualRegister(TRC);
11689 Register destOut =
MRI.createVirtualRegister(TRC);
11690 Register scratch =
MRI.createVirtualRegister(TRC);
11691 emitPostLd(BB, StartOfExit,
TII, dl, 1, scratch, srcIn, srcOut,
11692 IsThumb1, IsThumb2);
11693 emitPostSt(BB, StartOfExit,
TII, dl, 1, scratch, destIn, destOut,
11694 IsThumb1, IsThumb2);
11699 MI.eraseFromParent();
11711 "__chkstk is only supported on Windows");
11712 assert(Subtarget->
isThumb2() &&
"Windows on ARM requires Thumb-2 mode");
11732 switch (
TM.getCodeModel()) {
11774 MI.eraseFromParent();
11798 .
addReg(
MI.getOperand(0).getReg())
11806 MI.eraseFromParent();
11830 if (miI == BB->
end()) {
11832 if (Succ->isLiveIn(ARM::CPSR))
11838 SelectItr->addRegisterKilled(ARM::CPSR,
TRI);
11850 Register AddDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11851 BuildMI(TpEntry, Dl,
TII->get(ARM::t2ADDri), AddDestReg)
11857 Register LsrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11858 BuildMI(TpEntry, Dl,
TII->get(ARM::t2LSRri), LsrDestReg)
11864 Register TotalIterationsReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11865 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg)
11868 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopStart))
11869 .
addUse(TotalIterationsReg)
11876 return TotalIterationsReg;
11887 Register TotalIterationsReg,
bool IsMemcpy) {
11894 SrcPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11895 CurrSrcReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11896 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), SrcPhiReg)
11904 Register DestPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11905 Register CurrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11906 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), DestPhiReg)
11913 Register LoopCounterPhiReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11914 Register RemainingLoopIterationsReg =
11915 MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11916 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), LoopCounterPhiReg)
11917 .
addUse(TotalIterationsReg)
11919 .
addUse(RemainingLoopIterationsReg)
11923 Register PredCounterPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11924 Register RemainingElementsReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11925 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), PredCounterPhiReg)
11926 .
addUse(ElementCountReg)
11928 .
addUse(RemainingElementsReg)
11932 Register VccrReg =
MRI.createVirtualRegister(&ARM::VCCRRegClass);
11933 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VCTP8), VccrReg)
11934 .
addUse(PredCounterPhiReg)
11939 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2SUBri), RemainingElementsReg)
11940 .
addUse(PredCounterPhiReg)
11948 SrcValueReg =
MRI.createVirtualRegister(&ARM::MQPRRegClass);
11949 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VLDRBU8_post))
11958 SrcValueReg = OpSrcReg;
11960 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VSTRBU8_post))
11971 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg)
11972 .
addUse(LoopCounterPhiReg)
11975 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopEnd))
11976 .
addUse(RemainingLoopIterationsReg)
11989 bool isThumb2 = Subtarget->
isThumb2();
11990 switch (
MI.getOpcode()) {
11997 case ARM::tLDR_postidx: {
12001 .
add(
MI.getOperand(2))
12002 .
add(
MI.getOperand(3))
12003 .
add(
MI.getOperand(4))
12004 .
add(
MI.getOperand(0))
12006 MI.eraseFromParent();
12010 case ARM::MVE_MEMCPYLOOPINST:
12011 case ARM::MVE_MEMSETLOOPINST: {
12041 Register OpDestReg =
MI.getOperand(0).getReg();
12042 Register OpSrcReg =
MI.getOperand(1).getReg();
12043 Register OpSizeReg =
MI.getOperand(2).getReg();
12063 if (TpExit == BB) {
12065 "block containing memcpy/memset Pseudo");
12078 bool IsMemcpy =
MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
12080 OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy);
12095 MI.eraseFromParent();
12105 case ARM::t2STR_preidx:
12106 MI.setDesc(
TII->get(ARM::t2STR_PRE));
12108 case ARM::t2STRB_preidx:
12109 MI.setDesc(
TII->get(ARM::t2STRB_PRE));
12111 case ARM::t2STRH_preidx:
12112 MI.setDesc(
TII->get(ARM::t2STRH_PRE));
12115 case ARM::STRi_preidx:
12116 case ARM::STRBi_preidx: {
12117 unsigned NewOpc =
MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
12118 : ARM::STRB_PRE_IMM;
12120 unsigned Offset =
MI.getOperand(4).getImm();
12128 .
add(
MI.getOperand(0))
12129 .
add(
MI.getOperand(1))
12130 .
add(
MI.getOperand(2))
12132 .
add(
MI.getOperand(5))
12133 .
add(
MI.getOperand(6))
12135 MI.eraseFromParent();
12138 case ARM::STRr_preidx:
12139 case ARM::STRBr_preidx:
12140 case ARM::STRH_preidx: {
12142 switch (
MI.getOpcode()) {
12144 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG;
break;
12145 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG;
break;
12146 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE;
break;
12151 MI.eraseFromParent();
12155 case ARM::tMOVCCr_pseudo: {
12173 F->insert(It, copy0MBB);
12174 F->insert(It, sinkMBB);
12177 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
12183 if (!
MI.killsRegister(ARM::CPSR,
nullptr) &&
12199 .
addImm(
MI.getOperand(3).getImm())
12200 .
addReg(
MI.getOperand(4).getReg());
12215 .
addReg(
MI.getOperand(1).getReg())
12217 .
addReg(
MI.getOperand(2).getReg())
12220 MI.eraseFromParent();
12225 case ARM::BCCZi64: {
12231 bool RHSisZero =
MI.getOpcode() == ARM::BCCZi64;
12236 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12240 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12246 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12250 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12260 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12269 MI.eraseFromParent();
12273 case ARM::Int_eh_sjlj_setjmp:
12274 case ARM::Int_eh_sjlj_setjmp_nofp:
12275 case ARM::tInt_eh_sjlj_setjmp:
12276 case ARM::t2Int_eh_sjlj_setjmp:
12277 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12280 case ARM::Int_eh_sjlj_setup_dispatch:
12281 EmitSjLjDispatchBlock(
MI, BB);
12304 Fn->
insert(BBI, SinkBB);
12306 Register ABSSrcReg =
MI.getOperand(1).getReg();
12307 Register ABSDstReg =
MI.getOperand(0).getReg();
12308 bool ABSSrcKIll =
MI.getOperand(1).isKill();
12309 bool isThumb2 = Subtarget->
isThumb2();
12313 Register NewRsbDstReg =
MRI.createVirtualRegister(
12314 isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass);
12328 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12335 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).
addMBB(SinkBB)
12342 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
12351 TII->get(ARM::PHI), ABSDstReg)
12356 MI.eraseFromParent();
12361 case ARM::COPY_STRUCT_BYVAL_I32:
12363 return EmitStructByval(
MI, BB);
12364 case ARM::WIN__CHKSTK:
12365 return EmitLowered__chkstk(
MI, BB);
12366 case ARM::WIN__DBZCHK:
12367 return EmitLowered__dbzchk(
MI, BB);
12384 if (!
Node->hasAnyUseOfValue(0)) {
12385 MI.getOperand(0).setIsDead(
true);
12387 if (!
Node->hasAnyUseOfValue(1)) {
12388 MI.getOperand(1).setIsDead(
true);
12392 for (
unsigned I = 0;
I !=
MI.getOperand(4).getImm(); ++
I) {
12393 Register TmpReg =
MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
12394 : &ARM::GPRRegClass);
12401 if (
MI.getOpcode() == ARM::MEMCPY) {
12419 MCID = &
TII->get(NewOpc);
12422 MI.getDesc().getNumOperands() + 5 -
MI.getDesc().getSize()
12423 &&
"converted opcode should be the same except for cc_out"
12424 " (and, on Thumb1, pred)");
12434 MI.addOperand(
MI.getOperand(1));
12435 MI.removeOperand(1);
12439 for (
unsigned i =
MI.getNumOperands(); i--;) {
12441 if (
op.isReg() &&
op.isUse()) {
12444 MI.tieOperands(DefIdx, i);
12458 if (!
MI.hasOptionalDef() || !MCID->
operands()[ccOutIdx].isOptionalDef()) {
12459 assert(!NewOpc &&
"Optional cc_out operand required");
12464 bool definesCPSR =
false;
12465 bool deadCPSR =
false;
12466 for (
unsigned i = MCID->
getNumOperands(), e =
MI.getNumOperands(); i != e;
12470 definesCPSR =
true;
12473 MI.removeOperand(i);
12477 if (!definesCPSR) {
12478 assert(!NewOpc &&
"Optional cc_out operand required");
12481 assert(deadCPSR == !Node->hasAnyUseOfValue(1) &&
"inconsistent dead flag");
12483 assert(!
MI.getOperand(ccOutIdx).getReg() &&
12484 "expect uninitialized optional cc_out operand");
12522 switch (
N->getOpcode()) {
12523 default:
return false;
12525 CC =
N->getOperand(0);
12547 EVT VT =
N->getValueType(0);
12548 CC =
N->getOperand(0);
12549 if (
CC.getValueType() != MVT::i1 ||
CC.getOpcode() !=
ISD::SETCC)
12595 EVT VT =
N->getValueType(0);
12598 bool SwapSelectOps;
12600 NonConstantVal, DAG))
12606 OtherOp, NonConstantVal);
12612 CCOp, TrueVal, FalseVal);
12636 if (
N->getOpcode() ==
ARMISD::VTRN &&
N->getValueType(0) == MVT::v2i32)
12651 if (!
N->getValueType(0).is64BitVector())
12659 EVT VT =
N->getValueType(0);
12698 EVT VT =
N->getValueType(0);
12704 Opcode = Intrinsic::arm_neon_vpaddls;
12706 Opcode = Intrinsic::arm_neon_vpaddlu;
12734 EVT VT =
N->getValueType(0);
12749 unsigned nextIndex = 0;
12817 return DAG.
getNode(ExtOp, dl, VT, tmp);
12848 if (SRA.getOpcode() !=
ISD::SRA) {
12854 if (
auto Const = dyn_cast<ConstantSDNode>(SRA.getOperand(1))) {
12855 if (Const->getZExtValue() != 31)
12860 if (SRA.getOperand(0) !=
Mul)
12864 SDLoc dl(AddcNode);
12865 unsigned Opcode = 0;
12871 Op0 =
Mul.getOperand(0);
12872 Op1 =
Mul.getOperand(1);
12875 Op0 =
Mul.getOperand(0);
12876 Op1 =
Mul.getOperand(1).getOperand(0);
12879 Op0 =
Mul.getOperand(0).getOperand(0);
12880 Op1 =
Mul.getOperand(1);
12883 Op0 =
Mul->getOperand(0).getOperand(0);
12884 Op1 =
Mul->getOperand(1).getOperand(0);
12893 SDValue HiMLALResult(SMLAL.getNode(), 1);
12894 SDValue LoMLALResult(SMLAL.getNode(), 0);
12900 SDValue resNode(AddcNode, 0);
12929 "Expect an ADDE or SUBE");
12933 "ADDE node has the wrong inputs");
12952 "Expect ADDC with two result values. First: i32");
12972 bool IsLeftOperandMUL =
false;
12977 IsLeftOperandMUL =
true;
12988 SDValue *LowAddSub =
nullptr;
12991 if ((AddeSubeOp0 != MULOp.
getValue(1)) && (AddeSubeOp1 != MULOp.
getValue(1)))
12994 if (IsLeftOperandMUL)
12995 HiAddSub = &AddeSubeOp1;
12997 HiAddSub = &AddeSubeOp0;
13002 if (AddcSubcOp0 == MULOp.
getValue(0)) {
13003 LoMul = &AddcSubcOp0;
13004 LowAddSub = &AddcSubcOp1;
13006 if (AddcSubcOp1 == MULOp.
getValue(0)) {
13007 LoMul = &AddcSubcOp1;
13008 LowAddSub = &AddcSubcOp0;
13016 if (AddcSubcNode == HiAddSub->getNode() ||
13032 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->
useMulOps() &&
13046 return SDValue(AddeSubeNode, 0);
13057 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13067 return SDValue(AddeSubeNode, 0);
13079 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13088 SDNode *UmlalNode =
nullptr;
13113 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13127 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13132 SDNode* AddcNode =
N->getOperand(2).getNode();
13133 SDNode* AddeNode =
N->getOperand(3).getNode();
13141 {N->getOperand(0), N->getOperand(1),
13142 AddcNode->getOperand(0), AddcNode->getOperand(1)});
13166 int32_t imm =
C->getSExtValue();
13167 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13172 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
N->getOperand(0),
RHS);
13187 int64_t imm =
C->getSExtValue();
13198 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
13199 N->getOperand(0),
RHS,
N->getOperand(2));
13211 if (!Subtarget->hasMVEIntegerOps())
13224 SetCC =
N->getOperand(0);
13227 CC = cast<CondCodeSDNode>(SetCC->
getOperand(2))->get();
13228 TrueVal =
N->getOperand(1);
13229 FalseVal =
N->getOperand(2);
13231 LHS =
N->getOperand(0);
13232 RHS =
N->getOperand(1);
13233 CC = cast<CondCodeSDNode>(
N->getOperand(4))->get();
13234 TrueVal =
N->getOperand(2);
13235 FalseVal =
N->getOperand(3);
13240 unsigned int Opcode = 0;
13269 switch (TrueVal->getOpcode()) {
13288 if (TrueVal !=
LHS || FalseVal !=
RHS)
13291 EVT LeftType =
LHS->getValueType(0);
13292 EVT RightType =
RHS->getValueType(0);
13295 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13299 if (VectorScalarType != MVT::i32)
13307 if (VectorScalarType != MVT::i32)
13320 EVT VT =
N->getValueType(0);
13328 Shft =
N->getOperand(0);
13334 cast<CondCodeSDNode>(Cmp.getOperand(2))->get() !=
ISD::SETLT ||
13335 Cmp.getOperand(0) !=
N->getOperand(1) ||
13336 Cmp.getOperand(1) !=
N->getOperand(2))
13338 Shft =
N->getOperand(1);
13350 ScalarType = MVT::i8;
13353 case (1 << 15) - 1:
13354 ScalarType = MVT::i16;
13357 case (1ULL << 31) - 1:
13358 ScalarType = MVT::i32;
13389 unsigned LegalLanes = 128 / (ShftAmt + 1);
13413 for (
unsigned I = 0;
I < NumParts; ++
I) {
13430 if (!Subtarget->hasMVEIntegerOps())
13445 if (
N->getOperand(0).getOpcode() !=
ISD::XOR)
13455 if (!Const || !Const->isOne())
13473 EVT VT =
N->getValueType(0);
13475 if (!Subtarget->hasMVEIntegerOps() ||
13504 Opc = Intrinsic::arm_mve_vctp64;
13507 Opc = Intrinsic::arm_mve_vctp32;
13510 Opc = Intrinsic::arm_mve_vctp16;
13513 Opc = Intrinsic::arm_mve_vctp8;
13567 EVT VT =
N->getValueType(0);
13573 switch (
Op.getOpcode()) {
13590 !isa<ConstantSDNode>(N0) && N1->
hasOneUse()) {
13598 unsigned N0RedOp = 0;
13605 unsigned N1RedOp = 0;
13619 if (
SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13621 if (
SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13628 auto DistrubuteVecReduceLoad = [&](
SDValue N0,
SDValue N1,
bool IsForward) {
13642 LoadSDNode *Load0 = dyn_cast<LoadSDNode>(N0);
13643 LoadSDNode *Load1 = dyn_cast<LoadSDNode>(N1);
13652 if (!BaseLocDecomp0.getBase() ||
13653 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13654 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13656 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13658 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13668 if (IsBefore < 0) {
13671 }
else if (IsBefore > 0) {
13684 }
else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13694 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13704 if (
SDValue R = DistrubuteVecReduceLoad(N0, N1,
true))
13706 if (
SDValue R = DistrubuteVecReduceLoad(N1, N0,
false))
13713 if (!Subtarget->hasMVEIntegerOps())
13719 EVT VT =
N->getValueType(0);
13724 if (VT != MVT::i64)
13735 auto MakeVecReduce = [&](
unsigned Opcode,
unsigned OpcodeA,
SDValue NA,
13753 std::tie(Ops[0], Ops[1]) = DAG.
SplitScalar(NA, dl, MVT::i32, MVT::i32);
13755 unsigned S = VecRed->
getOpcode() == OpcodeA ? 2 : 0;
13804 "Expected shift op");
13822 if (
auto *Const = dyn_cast<ConstantSDNode>(N1->
getOperand(1))) {
13823 if (Const->getAPIntValue().ult(256))
13826 Const->getAPIntValue().sgt(-256))
13842 (
N->getOperand(0).getOpcode() ==
ISD::SHL ||
13843 N->getOperand(0).getOpcode() ==
ISD::SRL) &&
13844 "Expected XOR(SHIFT) pattern");
13847 auto *XorC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13848 auto *ShiftC = dyn_cast<ConstantSDNode>(
N->getOperand(0).getOperand(1));
13849 if (XorC && ShiftC) {
13850 unsigned MaskIdx, MaskLen;
13851 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13852 unsigned ShiftAmt = ShiftC->getZExtValue();
13853 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
13854 if (
N->getOperand(0).getOpcode() ==
ISD::SHL)
13855 return MaskIdx == ShiftAmt && MaskLen == (
BitWidth - ShiftAmt);
13856 return MaskIdx == 0 && MaskLen == (
BitWidth - ShiftAmt);
13866 N->getOperand(0).getOpcode() ==
ISD::SRL) ||
13868 N->getOperand(0).getOpcode() ==
ISD::SHL)) &&
13869 "Expected shift-shift mask");
13882 return Subtarget->hasMVEIntegerOps() &&
isTypeLegal(VT);
13886 if (!Subtarget->hasNEON()) {
13905 return Subtarget->hasFP64();
13908 return Subtarget->hasMVEFloatOps();
13937 if (ST->isThumb() && ST->isThumb1Only())
13941 for (
auto *U :
N->uses()) {
13942 switch(U->getOpcode()) {
13955 if (isa<ConstantSDNode>(U->getOperand(0)) ||
13956 isa<ConstantSDNode>(U->getOperand(1)))
13960 if (U->getOperand(0).getOpcode() ==
ISD::SHL ||
13961 U->getOperand(1).getOpcode() ==
ISD::SHL)
13971 if (
N->getOperand(0).getOpcode() !=
ISD::SHL)
13976 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13977 auto *C2 = dyn_cast<ConstantSDNode>(SHL.getOperand(1));
13978 if (!C1ShlC2 || !C2)
13981 APInt C2Int = C2->getAPIntValue();
13982 APInt C1Int = C1ShlC2->getAPIntValue();
13984 if (C2Int.
uge(C2Width))
13990 if ((C1Int & Mask) != C1Int)
13997 auto LargeImm = [](
const APInt &Imm) {
13998 unsigned Zeros = Imm.countl_zero() + Imm.countr_zero();
13999 return Imm.getBitWidth() - Zeros > 8;
14002 if (LargeImm(C1Int) || LargeImm(C2Int))
14014 SHL.dump();
N->dump());
14049 if (CSINC.getOpcode() !=
ARMISD::CSINC || !CSINC.hasOneUse())
14058 CSINC.getOperand(0)),
14059 CSINC.getOperand(1), CSINC.getOperand(2),
14060 CSINC.getOperand(3));
14079 if (!Subtarget->hasMVEIntegerOps() || !
N->getValueType(0).isVector())
14121 if (!Subtarget->hasVMLxForwarding())
14140 EVT VT =
N->getValueType(0);
14151 EVT VT =
N->getValueType(0);
14152 if (VT != MVT::v2i64)
14161 EVT VT = cast<VTSDNode>(
Op->getOperand(1))->getVT();
14163 return Op->getOperand(0);
14177 And =
And->getOperand(0);
14182 Mask = Mask->getOperand(0);
14185 Mask.getValueType() != MVT::v4i32)
14191 return And->getOperand(0);
14196 if (
SDValue Op0 = IsSignExt(N0)) {
14197 if (
SDValue Op1 = IsSignExt(N1)) {
14203 if (
SDValue Op0 = IsZeroExt(N0)) {
14204 if (
SDValue Op1 = IsZeroExt(N1)) {
14219 EVT VT =
N->getValueType(0);
14220 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14231 if (VT != MVT::i32)
14238 int64_t MulAmt =
C->getSExtValue();
14239 unsigned ShiftAmt = llvm::countr_zero<uint64_t>(MulAmt);
14241 ShiftAmt = ShiftAmt & (32 - 1);
14246 MulAmt >>= ShiftAmt;
14249 if (llvm::has_single_bit<uint32_t>(MulAmt - 1)) {
14257 }
else if (llvm::has_single_bit<uint32_t>(MulAmt + 1)) {
14269 if (llvm::has_single_bit<uint32_t>(MulAmtAbs + 1)) {
14277 }
else if (llvm::has_single_bit<uint32_t>(MulAmtAbs - 1)) {
14307 if (
N->getValueType(0) != MVT::i32)
14316 if (C1 == 255 || C1 == 65535)
14319 SDNode *N0 =
N->getOperand(0).getNode();
14333 if (!C2 || C2 >= 32)
14377 if (Trailing == C2 && C2 + C3 < 32) {
14390 if (Leading == C2 && C2 + C3 < 32) {
14418 EVT VT =
N->getValueType(0);
14422 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14425 APInt SplatBits, SplatUndef;
14426 unsigned SplatBitSize;
14428 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14429 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14430 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14431 SplatBitSize == 64) {
14465 if (!Subtarget->hasV6Ops() ||
14466 (Subtarget->isThumb() &&
14467 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14470 SDValue SRL = OR->getOperand(0);
14471 SDValue SHL = OR->getOperand(1);
14474 SRL = OR->getOperand(1);
14475 SHL = OR->getOperand(0);
14482 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) ||
14486 SDNode *SMULLOHI = SRL.getOperand(0).getNode();
14487 if (SRL.getOperand(0) !=
SDValue(SMULLOHI, 0) ||
14488 SHL.getOperand(0) !=
SDValue(SMULLOHI, 1))
14507 unsigned Opcode = 0;
14508 if (
isS16(OpS16, DAG))
14526 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6T2Ops())
14529 EVT VT =
N->getValueType(0);
14544 if (VT != MVT::i32)
14557 if (Mask == 0xffff)
14564 if ((Val & ~Mask) != Val)
14589 (Mask == ~Mask2)) {
14592 if (Subtarget->hasDSP() &&
14593 (Mask == 0xffff || Mask == 0xffff0000))
14606 (~Mask == Mask2)) {
14609 if (Subtarget->hasDSP() &&
14610 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14683 EVT VT =
N->getValueType(0);
14688 auto IsFreelyInvertable = [&](
SDValue V) {
14695 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14711 EVT VT =
N->getValueType(0);
14717 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14718 VT == MVT::v8i1 || VT == MVT::v16i1))
14721 APInt SplatBits, SplatUndef;
14722 unsigned SplatBitSize;
14724 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14725 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14726 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14727 SplatBitSize == 64) {
14763 unsigned SplatBitSize;
14766 APInt SplatBits0, SplatBits1;
14770 if (BVN0 && BVN0->
isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
14771 HasAnyUndefs) && !HasAnyUndefs) {
14772 if (BVN1 && BVN1->
isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
14773 HasAnyUndefs) && !HasAnyUndefs) {
14778 SplatBits0 == ~SplatBits1) {
14808 EVT VT =
N->getValueType(0);
14823 if (Subtarget->hasMVEIntegerOps()) {
14854 ToMask = ~N->getConstantOperandAPInt(2);
14860 isa<ConstantSDNode>(
From->getOperand(1))) {
14861 APInt Shift =
From->getConstantOperandAPInt(1);
14874 unsigned LastActiveBitInA =
A.countr_zero();
14875 unsigned FirstActiveBitInB =
B.getBitWidth() -
B.countl_zero() - 1;
14876 return LastActiveBitInA - 1 == FirstActiveBitInB;
14881 APInt ToMask, FromMask;
14889 APInt NewToMask, NewFromMask;
14891 if (NewFrom !=
From)
14895 if ((NewToMask & ToMask).getBoolValue())
14920 unsigned InvMask =
N->getConstantOperandVal(2);
14922 unsigned Width = llvm::bit_width<unsigned>(~InvMask) - LSB;
14924 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14925 "undefined behavior");
14926 unsigned Mask = (1u << Width) - 1;
14928 if ((Mask & (~Mask2)) == 0)
14930 N->getOperand(0), N1.
getOperand(0),
N->getOperand(2));
14937 APInt ToMask1, FromMask1;
14940 APInt ToMask2, FromMask2;
14946 APInt NewFromMask = FromMask1 | FromMask2;
14947 APInt NewToMask = ToMask1 | ToMask2;
14949 EVT VT =
N->getValueType(0);
14952 if (NewFromMask[0] == 0)
14964 APInt ToMask1 = ~N->getConstantOperandAPInt(2);
14965 APInt ToMask2 = ~N0.getConstantOperandAPInt(2);
14967 if (!N0.
hasOneUse() || (ToMask1 & ToMask2) != 0 ||
14971 EVT VT =
N->getValueType(0);
14974 N->getOperand(1),
N->getOperand(2));
14988 SDValue CSInc = Cmp->getOperand(0);
15038 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ)
15039 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15042 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
15044 N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15057 SDValue InDouble =
N->getOperand(0);
15066 !cast<LoadSDNode>(InNode)->isVolatile()) {
15072 SDValue BasePtr = LD->getBasePtr();
15074 DAG.
getLoad(MVT::i32,
DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
15075 LD->getAlign(), LD->getMemOperand()->getFlags());
15081 LD->getPointerInfo().getWithOffset(4),
15083 LD->getMemOperand()->getFlags());
15095 isa<ConstantSDNode>(InDouble.
getOperand(1))) {
15116 if (!Subtarget->
isLittle() && BVSwap)
15126 if (isa<ConstantSDNode>(BV.
getOperand(2))) {
15134 if (!Subtarget->
isLittle() && BVSwap)
15179 if (Copy.getValueType() == MVT::f32 &&
15181 bool HasGlue = Copy->getNumOperands() == 3;
15182 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1),
15183 HasGlue ? Copy->getOperand(2) :
SDValue()};
15184 EVT OutTys[] = {
N->getValueType(0), MVT::Other, MVT::Glue};
15202 if (
LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Op0)) {
15203 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15204 LN0->getMemoryVT() == MVT::i16) {
15207 LN0->getBasePtr(), LN0->getMemOperand());
15225 EVT VT =
N->getValueType(0);
15259 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
15260 for (
unsigned i = 0; i < NumElts; ++i) {
15261 SDNode *Elt =
N->getOperand(i).getNode();
15278 if (
N->getNumOperands() == 2)
15284 EVT VT =
N->getValueType(0);
15290 for (
unsigned i = 0; i < NumElts; ++i) {
15316 EVT VT =
N->getValueType(0);
15324 assert(EltVT == MVT::f32 &&
"Unexpected type!");
15329 Use->getValueType(0).isFloatingPoint())
15337 unsigned NumOfBitCastedElts = 0;
15339 unsigned NumOfRelevantElts = NumElts;
15340 for (
unsigned Idx = 0;
Idx < NumElts; ++
Idx) {
15345 ++NumOfBitCastedElts;
15346 }
else if (Elt.
isUndef() || isa<ConstantSDNode>(Elt))
15349 --NumOfRelevantElts;
15353 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15371 for (
unsigned Idx = 0 ;
Idx < NumElts; ++
Idx) {
15376 V->getOperand(0).getValueType() == MVT::i32)
15378 V = V.getOperand(0);
15395 EVT VT =
N->getValueType(0);
15402 if (
Op->getOperand(0).getValueType() == VT)
15403 return Op->getOperand(0);
15418 if (
Op.getValueType() == MVT::i32) {
15429 EVT VT =
N->getValueType(0);
15434 if (ST->isLittle())
15444 if (
Op->getOperand(0).getValueType() == VT)
15445 return Op->getOperand(0);
15454 if (!Subtarget->hasMVEIntegerOps())
15457 EVT VT =
N->getValueType(0);
15488 EVT VT =
N->getValueType(0);
15489 SDNode *Elt =
N->getOperand(1).getNode();
15504 Vec, V,
N->getOperand(2));
15514 EVT VT =
N->getValueType(0);
15523 Ext.getOperand(0).getValueType() == MVT::f32)
15524 Ext = Ext.getOperand(0);
15526 !isa<ConstantSDNode>(Ext.getOperand(1)) ||
15527 Ext.getConstantOperandVal(1) % 2 != 0)
15529 if (Ext->use_size() == 1 &&
15534 SDValue Op0 = Ext.getOperand(0);
15537 unsigned Lane = Ext.getConstantOperandVal(1);
15543 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15544 isa<ConstantSDNode>(V->getOperand(1)) &&
15545 V->getConstantOperandVal(1) == Lane + 1 &&
15546 V->getOperand(0).getResNo() == ResNo;
15548 if (OtherIt == Op0->
uses().end())
15553 SDValue OtherExt(*OtherIt, 0);
15566 DCI.
DAG.
getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32));
15578 EVT VT =
N->getValueType(0);
15584 if (VT == MVT::f16 &&
X.getValueType() == MVT::i32)
15586 if (VT == MVT::i32 &&
X.getValueType() == MVT::f16)
15588 if (VT == MVT::f32 &&
X.getValueType() == MVT::i32)
15591 while (
X.getValueType() != VT &&
X->getOpcode() ==
ISD::BITCAST)
15592 X =
X->getOperand(0);
15593 if (
X.getValueType() == VT)
15599 isa<ConstantSDNode>(
N->getOperand(1)) &&
15601 return Op0.
getOperand(
N->getConstantOperandVal(1));
15606 isa<ConstantSDNode>(
N->getOperand(1)) &&
15611 unsigned Offset =
N->getConstantOperandVal(1);
15623 unsigned Idx =
N->getConstantOperandVal(1);
15637 EVT VT =
N->getValueType(0);
15641 cast<VTSDNode>(
N->getOperand(1))->getVT() ==
15642 Op.getOperand(0).getValueType().getScalarType())
15652 SDValue SubVec =
N->getOperand(1);
15653 uint64_t IdxVal =
N->getConstantOperandVal(2);
15664 if (IdxVal == 0 && Vec.
isUndef())
15670 (IdxVal != 0 && IdxVal != NumSubElts))
15743 EVT VT =
N->getValueType(0);
15754 unsigned HalfElts = NumElts/2;
15756 for (
unsigned n = 0; n < NumElts; ++n) {
15759 if (MaskElt < (
int)HalfElts)
15761 else if (MaskElt >= (
int)NumElts && MaskElt < (
int)(NumElts + HalfElts))
15762 NewElt = HalfElts + MaskElt - NumElts;
15789 bool SimpleConstIncOnly,
15797 bool isLoadOp =
true;
15798 bool isLaneOp =
false;
15801 bool hasAlignment =
true;
15802 unsigned NewOpc = 0;
15803 unsigned NumVecs = 0;
15804 if (
Target.isIntrinsic) {
15805 unsigned IntNo =
N->getConstantOperandVal(1);
15809 case Intrinsic::arm_neon_vld1:
15813 case Intrinsic::arm_neon_vld2:
15817 case Intrinsic::arm_neon_vld3:
15821 case Intrinsic::arm_neon_vld4:
15825 case Intrinsic::arm_neon_vld1x2:
15828 hasAlignment =
false;
15830 case Intrinsic::arm_neon_vld1x3:
15833 hasAlignment =
false;
15835 case Intrinsic::arm_neon_vld1x4:
15838 hasAlignment =
false;
15840 case Intrinsic::arm_neon_vld2dup:
15844 case Intrinsic::arm_neon_vld3dup:
15848 case Intrinsic::arm_neon_vld4dup:
15852 case Intrinsic::arm_neon_vld2lane:
15857 case Intrinsic::arm_neon_vld3lane:
15862 case Intrinsic::arm_neon_vld4lane:
15867 case Intrinsic::arm_neon_vst1:
15872 case Intrinsic::arm_neon_vst2:
15877 case Intrinsic::arm_neon_vst3:
15882 case Intrinsic::arm_neon_vst4:
15887 case Intrinsic::arm_neon_vst2lane:
15893 case Intrinsic::arm_neon_vst3lane:
15899 case Intrinsic::arm_neon_vst4lane:
15905 case Intrinsic::arm_neon_vst1x2:
15909 hasAlignment =
false;
15911 case Intrinsic::arm_neon_vst1x3:
15915 hasAlignment =
false;
15917 case Intrinsic::arm_neon_vst1x4:
15921 hasAlignment =
false;
15926 switch (
N->getOpcode()) {
15962 VecTy =
N->getValueType(0);
15963 }
else if (
Target.isIntrinsic) {
15964 VecTy =
N->getOperand(
Target.AddrOpIdx + 1).getValueType();
15967 "Node has to be a load, a store, or an intrinsic!");
15968 VecTy =
N->getOperand(1).getValueType();
15976 if (isLaneOp || isVLDDUPOp)
15979 if (NumBytes >= 3 * 16 &&
User.ConstInc != NumBytes) {
15985 if (SimpleConstIncOnly &&
User.ConstInc != NumBytes)
15991 EVT AlignedVecTy = VecTy;
16008 if (isa<LSBaseSDNode>(
N)) {
16011 assert(NumVecs == 1 &&
"Unexpected multi-element generic load/store.");
16012 assert(!isLaneOp &&
"Unexpected generic load/store lane.");
16023 Alignment =
Align(1);
16029 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16031 for (n = 0; n < NumResultVecs; ++n)
16032 Tys[n] = AlignedVecTy;
16033 Tys[n++] = MVT::i32;
16034 Tys[n] = MVT::Other;
16049 unsigned LastOperand =
16050 hasAlignment ?
N->getNumOperands() - 1 :
N->getNumOperands();
16051 for (
unsigned i =
Target.AddrOpIdx + 1; i < LastOperand; ++i)
16060 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::STORE) {
16071 for (
unsigned i = 0; i < NumResultVecs; ++i)
16076 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::LOAD) {
16077 SDValue &LdVal = NewResults[0];
16113 switch (
N->getOpcode()) {
16116 if (isa<ConstantSDNode>(
N->getOperand(1))) {
16117 *
Ptr =
N->getOperand(0);
16118 *CInc =
N->getOperand(1);
16124 if (isa<ConstantSDNode>(
N->getOperand(2))) {
16125 *
Ptr =
N->getOperand(1);
16126 *CInc =
N->getOperand(2);
16161 const unsigned AddrOpIdx = ((isIntrinsic ||
isStore) ? 2 : 1);
16170 UE =
Addr.getNode()->use_end(); UI != UE; ++UI) {
16172 if (UI.getUse().getResNo() !=
Addr.getResNo() ||
16177 unsigned ConstInc =
16195 if (UI.getUse().getResNo() !=
Base.getResNo() ||
User ==
Addr.getNode() ||
16200 unsigned UserOffset =
16203 if (!UserOffset || UserOffset <=
Offset)
16206 unsigned NewConstInc = UserOffset -
Offset;
16216 unsigned NumValidUpd = BaseUpdates.
size();
16217 for (
unsigned I = 0;
I < NumValidUpd;) {
16221 std::swap(BaseUpdates[
I], BaseUpdates[NumValidUpd]);
16229 BaseUpdates.
resize(NumValidUpd);
16234 std::stable_sort(BaseUpdates.
begin(), BaseUpdates.
end(),
16236 return LHS.ConstInc < RHS.ConstInc;
16265 unsigned IntNo =
N->getConstantOperandVal(1);
16266 if (IntNo == Intrinsic::arm_mve_vst2q &&
N->getConstantOperandVal(5) != 1)
16268 if (IntNo == Intrinsic::arm_mve_vst4q &&
N->getConstantOperandVal(7) != 3)
16273 UE =
Addr.getNode()->use_end();
16277 UI.getUse().getResNo() !=
Addr.getResNo())
16293 bool isLoadOp =
true;
16294 unsigned NewOpc = 0;
16295 unsigned NumVecs = 0;
16299 case Intrinsic::arm_mve_vld2q:
16303 case Intrinsic::arm_mve_vld4q:
16307 case Intrinsic::arm_mve_vst2q:
16312 case Intrinsic::arm_mve_vst4q:
16322 VecTy =
N->getValueType(0);
16324 VecTy =
N->getOperand(3).getValueType();
16338 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16340 for (n = 0; n < NumResultVecs; ++n)
16342 Tys[n++] = MVT::i32;
16343 Tys[n] = MVT::Other;
16352 for (
unsigned i = 3; i <
N->getNumOperands(); ++i)
16360 for (
unsigned i = 0; i < NumResultVecs; ++i)
16379 EVT VT =
N->getValueType(0);
16385 SDNode *VLD =
N->getOperand(0).getNode();
16388 unsigned NumVecs = 0;
16389 unsigned NewOpc = 0;
16391 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16394 }
else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16397 }
else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16410 if (UI.getUse().getResNo() == NumVecs)
16414 VLDLaneNo !=
User->getConstantOperandVal(1))
16421 for (n = 0; n < NumVecs; ++n)
16423 Tys[n] = MVT::Other;
16434 unsigned ResNo = UI.getUse().
getResNo();
16436 if (ResNo == NumVecs)
16444 std::vector<SDValue> VLDDupResults;
16445 for (
unsigned n = 0; n < NumVecs; ++n)
16459 EVT VT =
N->getValueType(0);
16462 if (Subtarget->hasMVEIntegerOps()) {
16466 ExtractVT = MVT::i32;
16468 N->getOperand(0),
N->getOperand(1));
16480 Op =
Op.getOperand(0);
16485 unsigned EltSize =
Op.getScalarValueSizeInBits();
16487 unsigned Imm =
Op.getConstantOperandVal(0);
16503 if (Subtarget->hasMVEIntegerOps()) {
16506 if (
Op.getValueType() == MVT::f32)
16509 else if (
Op.getValueType() == MVT::f16)
16514 if (!Subtarget->hasNEON())
16521 if (LD &&
Op.hasOneUse() && LD->isUnindexed() &&
16522 LD->getMemoryVT() ==
N->getValueType(0).getVectorElementType()) {
16523 SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1),
16528 LD->getMemoryVT(), LD->getMemOperand());
16539 EVT VT =
N->getValueType(0);
16561 assert(StVT != VT &&
"Cannot truncate to the same type");
16571 if (0 != (NumElems * FromEltSz) % ToEltSz)
16574 unsigned SizeRatio = FromEltSz / ToEltSz;
16579 NumElems * SizeRatio);
16585 for (
unsigned i = 0; i < NumElems; ++i)
16599 MVT StoreType = MVT::i8;
16601 if (TLI.
isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16620 unsigned E = (ToEltSz * NumElems) / StoreType.
getSizeInBits();
16621 for (
unsigned I = 0;
I < E;
I++) {
16652 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16655 unsigned NumElements = 4;
16672 unsigned Off0 = Rev ? NumElts : 0;
16673 unsigned Off1 = Rev ? 0 : NumElts;
16675 for (
unsigned I = 0;
I < NumElts;
I += 2) {
16676 if (M[
I] >= 0 && M[
I] != (
int)(Off0 +
I / 2))
16678 if (M[
I + 1] >= 0 && M[
I + 1] != (
int)(Off1 +
I / 2))
16685 if (
auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Trunc.
getOperand(0)))
16686 if (isVMOVNShuffle(Shuffle,
false) || isVMOVNShuffle(Shuffle,
true))
16706 unsigned NewOffset = i * NumElements * ToEltVT.
getSizeInBits() / 8;
16721 NewToVT, Alignment, MMOFlags, AAInfo);
16754 unsigned NewOffset =
16762 NewToVT, Alignment, MMOFlags, AAInfo);
16784 {Extract.getOperand(0), Extract.getOperand(1)});
16815 if (Subtarget->hasNEON())
16819 if (Subtarget->hasMVEFloatOps())
16823 if (Subtarget->hasMVEIntegerOps()) {
16899 if (!Subtarget->hasNEON())
16903 if (!
Op.getValueType().isVector() || !
Op.getValueType().isSimple() ||
16908 if (!isa<BuildVectorSDNode>(ConstVec))
16911 MVT FloatTy =
Op.getSimpleValueType().getVectorElementType();
16913 MVT IntTy =
N->getSimpleValueType(0).getVectorElementType();
16915 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
16916 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
16927 if (
C == -1 ||
C == 0 ||
C > 32)
16932 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
16933 Intrinsic::arm_neon_vcvtfp2fxu;
16936 DAG.
getConstant(IntrinsicOpcode, dl, MVT::i32),
Op->getOperand(0),
16939 if (IntBits < FloatBits)
16947 if (!Subtarget->hasMVEFloatOps())
16955 EVT VT =
N->getValueType(0);
16960 auto isIdentitySplat = [&](
SDValue Op,
bool NSZ) {
16964 uint64_t ImmVal =
Op.getOperand(0).getConstantOperandVal(0);
16965 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
16967 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
16980 if (!isIdentitySplat(Op1.
getOperand(2), NSZ))
16991 EVT VT =
N->getValueType(0);
16994 if (!
N->getFlags().hasAllowReassociation())
17001 unsigned Opc =
A.getConstantOperandVal(0);
17002 if (Opc != Intrinsic::arm_mve_vcmlaq)
17007 A.getOperand(3),
A.getOperand(4));
17039 if (!Subtarget->hasNEON())
17043 unsigned OpOpcode =
Op.getNode()->getOpcode();
17044 if (!
N->getValueType(0).isVector() || !
N->getValueType(0).isSimple() ||
17048 SDValue ConstVec =
N->getOperand(1);
17049 if (!isa<BuildVectorSDNode>(ConstVec))
17052 MVT FloatTy =
N->getSimpleValueType(0).getVectorElementType();
17054 MVT IntTy =
Op.getOperand(0).getSimpleValueType().getVectorElementType();
17056 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
17057 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17077 int32_t
C = IntVal.exactLogBase2();
17078 if (
C == -1 ||
C == 0 ||
C > 32)
17084 if (IntBits < FloatBits)
17086 NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, ConvInput);
17088 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfxs2fp
17089 : Intrinsic::arm_neon_vcvtfxu2fp;
17097 if (!ST->hasMVEIntegerOps())
17101 EVT ResVT =
N->getValueType(0);
17129 EVT AVT =
A.getValueType();
17135 auto ExtendIfNeeded = [&](
SDValue A,
unsigned ExtendCode) {
17136 EVT AVT =
A.getValueType();
17148 if (ExtTypeMatches(
A, ExtTypes))
17149 return ExtendIfNeeded(
A, ExtendCode);
17152 auto IsPredVADDV = [&](
MVT RetTy,
unsigned ExtendCode,
17159 if (Ext->getOpcode() != ExtendCode)
17162 if (ExtTypeMatches(
A, ExtTypes))
17163 return ExtendIfNeeded(
A, ExtendCode);
17176 if (ResVT !=
RetTy)
17179 if (
Mul->getOpcode() == ExtendCode &&
17180 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17182 Mul =
Mul->getOperand(0);
17191 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17192 A = ExtendIfNeeded(
A, ExtendCode);
17193 B = ExtendIfNeeded(
B, ExtendCode);
17211 if (
Mul->getOpcode() == ExtendCode &&
17212 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17214 Mul =
Mul->getOperand(0);
17223 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17224 A = ExtendIfNeeded(
A, ExtendCode);
17225 B = ExtendIfNeeded(
B, ExtendCode);
17236 EVT VT = Ops[0].getValueType();
17237 if (VT == MVT::v16i8) {
17239 "Unexpected illegal long reduction opcode");
17244 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
17247 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
17268 if (IsVMLAV(MVT::i64,
ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17271 if (IsVMLAV(MVT::i64,
ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17335 Op =
Op->getOperand(1);
17337 Op->getOperand(0)->getOpcode() ==
ISD::MUL) {
17339 if (
Mul->getOperand(0) ==
Mul->getOperand(1) &&
17356 unsigned VecOp =
N->getOperand(0).getValueType().isVector() ? 0 : 2;
17357 auto *Shuf = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp));
17358 if (!Shuf || !Shuf->getOperand(1).isUndef())
17363 APInt SetElts(Mask.size(), 0);
17364 for (
int E : Mask) {
17365 if (E < 0 || E >= (
int)Mask.size())
17372 if (
N->getNumOperands() != VecOp + 1) {
17373 auto *Shuf2 = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp + 1));
17374 if (!Shuf2 || !Shuf2->getOperand(1).isUndef() || Shuf2->getMask() != Mask)
17380 if (
Op.getValueType().isVector())
17392 unsigned IsTop =
N->getConstantOperandVal(2);
17399 if (Op0->
isUndef() && !IsTop)
17413 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17415 APInt Op0DemandedElts =
17416 IsTop ? Op1DemandedElts
17431 unsigned IsTop =
N->getConstantOperandVal(2);
17433 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17434 APInt Op0DemandedElts =
17446 EVT VT =
N->getValueType(0);
17450 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(
LHS);
17451 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(
RHS);
17453 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
17454 LHS.getOperand(1).isUndef() &&
RHS.getOperand(1).isUndef() &&
17458 LHS.getOperand(0),
RHS.getOperand(0));
17472 if (
auto C = dyn_cast<ConstantSDNode>(
N->getOperand(2))) {
17473 int ShiftAmt =
C->getSExtValue();
17474 if (ShiftAmt == 0) {
17480 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17481 unsigned NewOpcode =
17497 unsigned IntNo =
N->getConstantOperandVal(0);
17508 case Intrinsic::arm_neon_vshifts:
17509 case Intrinsic::arm_neon_vshiftu:
17510 case Intrinsic::arm_neon_vrshifts:
17511 case Intrinsic::arm_neon_vrshiftu:
17512 case Intrinsic::arm_neon_vrshiftn:
17513 case Intrinsic::arm_neon_vqshifts:
17514 case Intrinsic::arm_neon_vqshiftu:
17515 case Intrinsic::arm_neon_vqshiftsu:
17516 case Intrinsic::arm_neon_vqshiftns:
17517 case Intrinsic::arm_neon_vqshiftnu:
17518 case Intrinsic::arm_neon_vqshiftnsu:
17519 case Intrinsic::arm_neon_vqrshiftns:
17520 case Intrinsic::arm_neon_vqrshiftnu:
17521 case Intrinsic::arm_neon_vqrshiftnsu: {
17522 EVT VT =
N->getOperand(1).getValueType();
17524 unsigned VShiftOpc = 0;
17527 case Intrinsic::arm_neon_vshifts:
17528 case Intrinsic::arm_neon_vshiftu:
17533 if (
isVShiftRImm(
N->getOperand(2), VT,
false,
true, Cnt)) {
17540 case Intrinsic::arm_neon_vrshifts:
17541 case Intrinsic::arm_neon_vrshiftu:
17546 case Intrinsic::arm_neon_vqshifts:
17547 case Intrinsic::arm_neon_vqshiftu:
17552 case Intrinsic::arm_neon_vqshiftsu:
17557 case Intrinsic::arm_neon_vrshiftn:
17558 case Intrinsic::arm_neon_vqshiftns:
17559 case Intrinsic::arm_neon_vqshiftnu:
17560 case Intrinsic::arm_neon_vqshiftnsu:
17561 case Intrinsic::arm_neon_vqrshiftns:
17562 case Intrinsic::arm_neon_vqrshiftnu:
17563 case Intrinsic::arm_neon_vqrshiftnsu:
17575 case Intrinsic::arm_neon_vshifts:
17576 case Intrinsic::arm_neon_vshiftu:
17579 case Intrinsic::arm_neon_vrshifts:
17582 case Intrinsic::arm_neon_vrshiftu:
17585 case Intrinsic::arm_neon_vrshiftn:
17588 case Intrinsic::arm_neon_vqshifts:
17591 case Intrinsic::arm_neon_vqshiftu:
17594 case Intrinsic::arm_neon_vqshiftsu:
17597 case Intrinsic::arm_neon_vqshiftns:
17600 case Intrinsic::arm_neon_vqshiftnu:
17603 case Intrinsic::arm_neon_vqshiftnsu:
17606 case Intrinsic::arm_neon_vqrshiftns:
17609 case Intrinsic::arm_neon_vqrshiftnu:
17612 case Intrinsic::arm_neon_vqrshiftnsu:
17618 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17619 N->getOperand(1), DAG.
getConstant(Cnt, dl, MVT::i32));
17622 case Intrinsic::arm_neon_vshiftins: {
17623 EVT VT =
N->getOperand(1).getValueType();
17625 unsigned VShiftOpc = 0;
17629 else if (
isVShiftRImm(
N->getOperand(3), VT,
false,
true, Cnt))
17636 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17637 N->getOperand(1),
N->getOperand(2),
17641 case Intrinsic::arm_neon_vqrshifts:
17642 case Intrinsic::arm_neon_vqrshiftu:
17646 case Intrinsic::arm_mve_vqdmlah:
17647 case Intrinsic::arm_mve_vqdmlash:
17648 case Intrinsic::arm_mve_vqrdmlah:
17649 case Intrinsic::arm_mve_vqrdmlash:
17650 case Intrinsic::arm_mve_vmla_n_predicated:
17651 case Intrinsic::arm_mve_vmlas_n_predicated:
17652 case Intrinsic::arm_mve_vqdmlah_predicated:
17653 case Intrinsic::arm_mve_vqdmlash_predicated:
17654 case Intrinsic::arm_mve_vqrdmlah_predicated:
17655 case Intrinsic::arm_mve_vqrdmlash_predicated: {
17660 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
17667 case Intrinsic::arm_mve_minv:
17668 case Intrinsic::arm_mve_maxv:
17669 case Intrinsic::arm_mve_minav:
17670 case Intrinsic::arm_mve_maxav:
17671 case Intrinsic::arm_mve_minv_predicated:
17672 case Intrinsic::arm_mve_maxv_predicated:
17673 case Intrinsic::arm_mve_minav_predicated:
17674 case Intrinsic::arm_mve_maxav_predicated: {
17677 unsigned BitWidth =
N->getOperand(2)->getValueType(0).getScalarSizeInBits();
17684 case Intrinsic::arm_mve_addv: {
17687 bool Unsigned =
N->getConstantOperandVal(2);
17692 case Intrinsic::arm_mve_addlv:
17693 case Intrinsic::arm_mve_addlv_predicated: {
17696 bool Unsigned =
N->getConstantOperandVal(2);
17697 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
17702 for (
unsigned i = 1, e =
N->getNumOperands(); i < e; i++)
17725 EVT VT =
N->getValueType(0);
17727 if (ST->isThumb1Only() &&
N->getOpcode() ==
ISD::SHL && VT == MVT::i32 &&
17728 N->getOperand(0)->getOpcode() ==
ISD::AND &&
17729 N->getOperand(0)->hasOneUse()) {
17737 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17746 if (AndMask == 255 || AndMask == 65535)
17750 if (MaskedBits > ShiftAmt) {
17765 if (ST->hasMVEIntegerOps())
17770 switch (
N->getOpcode()) {
17783 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
17784 unsigned VShiftOpc =
17787 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
17803 if (!LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed() ||
17806 EVT FromVT = LD->getValueType(0);
17807 EVT ToVT =
N->getValueType(0);
17814 unsigned NumElements = 0;
17815 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
17817 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
17819 if (NumElements == 0 ||
17829 SDValue BasePtr = LD->getBasePtr();
17830 Align Alignment = LD->getOriginalAlign();
17851 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
17852 Alignment, MMOFlags, AAInfo);
17858 if (FromEltVT == MVT::f16) {
17861 for (
unsigned i = 0; i < Loads.
size(); i++) {
17887 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
17891 EVT VT =
N->getValueType(0);
17895 if (VT == MVT::i32 &&
17896 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
17898 isa<ConstantSDNode>(Lane)) {
17901 switch (
N->getOpcode()) {
17915 if (ST->hasMVEIntegerOps())
17924 if (ST->hasMVEFloatOps())
17935 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
17939 EVT VT =
Op.getValueType();
17942 if (VT != MVT::i32 ||
17944 !isa<ConstantSDNode>(
Op.getOperand(1)) ||
17955 APInt MaxC = Max.getConstantOperandAPInt(1);
17958 !(MinC + 1).isPowerOf2())
17976 EVT VT =
N->getValueType(0);
17979 if (VT == MVT::i32)
17982 if (!ST->hasMVEIntegerOps())
17988 if (VT != MVT::v4i32 && VT != MVT::v8i16)
17991 auto IsSignedSaturate = [&](
SDNode *Min,
SDNode *Max) {
17999 if (VT == MVT::v4i32)
18000 SaturateC =
APInt(32, (1 << 15) - 1,
true);
18002 SaturateC =
APInt(16, (1 << 7) - 1,
true);
18009 MaxC != ~SaturateC)
18014 if (IsSignedSaturate(
N, N0.
getNode())) {
18017 if (VT == MVT::v4i32) {
18018 HalfVT = MVT::v8i16;
18019 ExtVT = MVT::v4i16;
18021 HalfVT = MVT::v16i8;
18036 auto IsUnsignedSaturate = [&](
SDNode *Min) {
18042 if (VT == MVT::v4i32)
18043 SaturateC =
APInt(32, (1 << 16) - 1,
true);
18045 SaturateC =
APInt(16, (1 << 8) - 1,
true);
18054 if (IsUnsignedSaturate(
N)) {
18058 if (VT == MVT::v4i32) {
18059 HalfVT = MVT::v8i16;
18060 ExtConst = 0x0000FFFF;
18062 HalfVT = MVT::v16i8;
18084 const APInt *CV = &
C->getAPIntValue();
18102 SDValue Op0 = CMOV->getOperand(0);
18103 SDValue Op1 = CMOV->getOperand(1);
18104 auto CC = CMOV->getConstantOperandAPInt(2).getLimitedValue();
18105 SDValue CmpZ = CMOV->getOperand(4);
18141 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18148 if ((OrCI & Known.
Zero) != OrCI)
18154 EVT VT =
X.getValueType();
18155 unsigned BitInX = AndC->
logBase2();
18163 for (
unsigned BitInY = 0, NumActiveBits = OrCI.
getActiveBits();
18164 BitInY < NumActiveBits; ++BitInY) {
18165 if (OrCI[BitInY] == 0)
18168 Mask.setBit(BitInY);
18185 switch (
N->getOpcode()) {
18189 if (!isa<ConstantSDNode>(
N.getOperand(1)))
18191 if (!cast<ConstantSDNode>(
N.getOperand(1))->isOne())
18197 auto *Const = dyn_cast<ConstantSDNode>(
N.getOperand(1));
18200 if (Const->isZero())
18202 else if (Const->isOne())
18206 CC = cast<CondCodeSDNode>(
N.getOperand(2))->get();
18210 unsigned IntOp =
N.getConstantOperandVal(1);
18211 if (IntOp != Intrinsic::test_start_loop_iterations &&
18212 IntOp != Intrinsic::loop_decrement_reg)
18238 bool Negate =
false;
18244 Cond =
N->getOperand(1);
18245 Dest =
N->getOperand(2);
18248 CC = cast<CondCodeSDNode>(
N->getOperand(1))->get();
18249 Cond =
N->getOperand(2);
18250 Dest =
N->getOperand(4);
18251 if (
auto *Const = dyn_cast<ConstantSDNode>(
N->getOperand(3))) {
18252 if (!Const->isOne() && !Const->isZero())
18254 Imm = Const->getZExtValue();
18282 assert((IsTrueIfZero(
CC, Imm) || IsFalseIfZero(
CC, Imm)) &&
18283 "unsupported condition");
18288 unsigned IntOp =
Int->getConstantOperandVal(1);
18290 &&
"expected single br user");
18291 SDNode *Br = *
N->use_begin();
18301 if (IntOp == Intrinsic::test_start_loop_iterations) {
18305 if (IsTrueIfZero(
CC, Imm)) {
18306 SDValue Ops[] = {Chain, Setup, Dest};
18311 UpdateUncondBr(Br, Dest, DAG);
18313 SDValue Ops[] = {Chain, Setup, OtherTarget};
18326 DAG.
getVTList(MVT::i32, MVT::Other), Args);
18334 if (
Target == OtherTarget)
18335 UpdateUncondBr(Br, Dest, DAG);
18354 EVT VT =
N->getValueType(0);
18373 LHS->getOperand(0)->getOperand(3),
LHS->getOperand(0)->getOperand(4));
18387 EVT VT =
N->getValueType(0);
18391 SDValue FalseVal =
N->getOperand(0);
18392 SDValue TrueVal =
N->getOperand(1);
18397 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops()) {
18423 N->getOperand(3), Cmp);
18428 N->getOperand(3), NewCmp);
18437 LHS->getOperand(2),
LHS->getOperand(3),
18438 LHS->getOperand(4));
18448 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ ||
18452 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
18454 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
18457 N->getOperand(3),
C);
18464 if (!Subtarget->
isThumb1Only() && Subtarget->hasV5TOps()) {
18500 N->getOperand(3), CPSRGlue.
getValue(1));
18515 N->getOperand(3), CPSRGlue.
getValue(1));
18533 const APInt *TrueConst;
18535 ((FalseVal.getOpcode() ==
ARMISD::SUBC && FalseVal.getOperand(0) ==
LHS &&
18536 FalseVal.getOperand(1) ==
RHS) ||
18540 unsigned ShiftAmount = TrueConst->
logBase2();
18555 if (Known.
Zero == 0xfffffffe)
18558 else if (Known.
Zero == 0xffffff00)
18561 else if (Known.
Zero == 0xffff0000)
18574 EVT DstVT =
N->getValueType(0);
18577 if (ST->hasMVEIntegerOps() && Src.getOpcode() ==
ARMISD::VDUP) {
18578 EVT SrcVT = Src.getValueType();
18586 Src = Src.getOperand(0);
18590 EVT SrcVT = Src.getValueType();
18610 EVT VT =
N->getValueType(0);
18618 if (
N->getNumOperands() == 2 &&
18622 N->getOperand(0).getOperand(1),
18623 N->getOperand(1).getOperand(0),
18624 N->getOperand(1).getOperand(1));
18627 if (
N->getNumOperands() == 2 &&
18630 auto *S0 = cast<ShuffleVectorSDNode>(
N->getOperand(0).getNode());
18631 auto *
S1 = cast<ShuffleVectorSDNode>(
N->getOperand(1).getNode());
18633 if (S0->getOperand(0) ==
S1->getOperand(0) &&
18634 S0->getOperand(1) ==
S1->getOperand(1)) {
18637 Mask.append(
S1->getMask().begin(),
S1->getMask().end());
18657 return Op.getOpcode() == ISD::BUILD_VECTOR ||
18658 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
18659 (Op.getOpcode() == ISD::BITCAST &&
18660 Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR);
18663 for (
unsigned Op = 0;
Op <
N->getNumOperands();
Op++) {
18665 for (
unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
18682 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18683 int NumIns =
N->getNumOperands();
18684 assert((NumIns == 2 || NumIns == 4) &&
18685 "Expected 2 or 4 inputs to an MVETrunc");
18687 if (
N->getNumOperands() == 4)
18691 for (
int I = 0;
I < NumIns;
I++) {
18693 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18713 if (!LD || !LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed())
18716 EVT FromVT = LD->getMemoryVT();
18717 EVT ToVT =
N->getValueType(0);
18724 unsigned NumElements = 0;
18725 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
18727 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
18729 assert(NumElements != 0);
18735 LD->getExtensionType() != NewExtType)
18742 SDValue BasePtr = LD->getBasePtr();
18743 Align Alignment = LD->getOriginalAlign();
18762 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
18763 Alignment, MMOFlags, AAInfo);
18779 EVT VT =
N->getValueType(0);
18781 assert(
N->getNumValues() == 2 &&
"Expected MVEEXT with 2 elements");
18782 assert((VT == MVT::v4i32 || VT == MVT::v8i16) &&
"Unexpected MVEEXT type");
18784 EVT ExtVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18786 auto Extend = [&](
SDValue V) {
18796 SDValue Ext = Extend(
N->getOperand(0));
18801 if (
auto *SVN = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(0))) {
18804 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
18809 auto CheckInregMask = [&](
int Start,
int Offset) {
18817 if (CheckInregMask(0, 0))
18819 else if (CheckInregMask(0, 1))
18820 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18821 else if (CheckInregMask(0, Mask.size()))
18823 else if (CheckInregMask(0, Mask.size() + 1))
18824 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18829 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18833 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18840 if (
N->getOperand(0)->getOpcode() ==
ISD::LOAD)
18850 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18851 int NumOuts =
N->getNumValues();
18852 assert((NumOuts == 2 || NumOuts == 4) &&
18853 "Expected 2 or 4 outputs to an MVEEXT");
18854 EVT LoadVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18856 if (
N->getNumOperands() == 4)
18862 StackPtr, MPI,
Align(4));
18865 for (
int I = 0;
I < NumOuts;
I++) {
18867 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18868 DAG.
getConstant(
I * 16 / NumOuts,
DL, StackPtr.getValueType()));
18873 VT, Chain,
Ptr, MPI, LoadVT,
Align(4));
18882 switch (
N->getOpcode()) {
18999 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19006 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19017 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19025 unsigned LowWidth =
N->getOperand(0).getValueType().getSizeInBits();
19027 unsigned HighWidth =
N->getOperand(1).getValueType().getSizeInBits();
19035 unsigned HighWidth =
N->getOperand(0).getValueType().getSizeInBits();
19037 unsigned LowWidth =
N->getOperand(1).getValueType().getSizeInBits();
19045 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19056 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19065 switch (
N->getConstantOperandVal(1)) {
19066 case Intrinsic::arm_neon_vld1:
19067 case Intrinsic::arm_neon_vld1x2:
19068 case Intrinsic::arm_neon_vld1x3:
19069 case Intrinsic::arm_neon_vld1x4:
19070 case Intrinsic::arm_neon_vld2:
19071 case Intrinsic::arm_neon_vld3:
19072 case Intrinsic::arm_neon_vld4:
19073 case Intrinsic::arm_neon_vld2lane:
19074 case Intrinsic::arm_neon_vld3lane:
19075 case Intrinsic::arm_neon_vld4lane:
19076 case Intrinsic::arm_neon_vld2dup:
19077 case Intrinsic::arm_neon_vld3dup:
19078 case Intrinsic::arm_neon_vld4dup:
19079 case Intrinsic::arm_neon_vst1:
19080 case Intrinsic::arm_neon_vst1x2:
19081 case Intrinsic::arm_neon_vst1x3:
19082 case Intrinsic::arm_neon_vst1x4:
19083 case Intrinsic::arm_neon_vst2:
19084 case Intrinsic::arm_neon_vst3:
19085 case Intrinsic::arm_neon_vst4:
19086 case Intrinsic::arm_neon_vst2lane:
19087 case Intrinsic::arm_neon_vst3lane:
19088 case Intrinsic::arm_neon_vst4lane:
19090 case Intrinsic::arm_mve_vld2q:
19091 case Intrinsic::arm_mve_vld4q:
19092 case Intrinsic::arm_mve_vst2q:
19093 case Intrinsic::arm_mve_vst4q:
19110 unsigned *
Fast)
const {
19119 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19121 if (AllowsUnaligned) {
19123 *
Fast = Subtarget->hasV7Ops();
19128 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19132 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->
isLittle())) {
19139 if (!Subtarget->hasMVEIntegerOps())
19143 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19144 Ty == MVT::v2i1)) {
19152 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19168 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19169 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19170 Ty == MVT::v2f64) {
19183 if ((
Op.isMemcpy() ||
Op.isZeroMemset()) && Subtarget->hasNEON() &&
19184 !FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat)) {
19186 if (
Op.size() >= 16 &&
19192 }
else if (
Op.size() >= 8 &&
19213 return (SrcBits == 64 && DestBits == 32);
19222 return (SrcBits == 64 && DestBits == 32);
19258 return Subtarget->hasFullFP16();
19268 return Ext->getType()->getScalarSizeInBits() ==
19269 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
19274 !areExtDoubled(cast<Instruction>(Ext1)) ||
19275 !areExtDoubled(cast<Instruction>(Ext2)))
19286 if (!
I->getType()->isVectorTy())
19289 if (Subtarget->hasNEON()) {
19290 switch (
I->getOpcode()) {
19291 case Instruction::Sub:
19292 case Instruction::Add: {
19304 if (!Subtarget->hasMVEIntegerOps())
19308 if (!
I->hasOneUse())
19310 auto *Sub = cast<Instruction>(*
I->users().begin());
19311 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(1) ==
I;
19321 switch (
I->getOpcode()) {
19322 case Instruction::Add:
19323 case Instruction::Mul:
19324 case Instruction::FAdd:
19325 case Instruction::ICmp:
19326 case Instruction::FCmp:
19328 case Instruction::FMul:
19329 return !IsFMSMul(
I);
19330 case Instruction::Sub:
19331 case Instruction::FSub:
19332 case Instruction::Shl:
19333 case Instruction::LShr:
19334 case Instruction::AShr:
19335 return Operand == 1;
19336 case Instruction::Call:
19337 if (
auto *
II = dyn_cast<IntrinsicInst>(
I)) {
19338 switch (
II->getIntrinsicID()) {
19339 case Intrinsic::fma:
19341 case Intrinsic::sadd_sat:
19342 case Intrinsic::uadd_sat:
19343 case Intrinsic::arm_mve_add_predicated:
19344 case Intrinsic::arm_mve_mul_predicated:
19345 case Intrinsic::arm_mve_qadd_predicated:
19346 case Intrinsic::arm_mve_vhadd:
19347 case Intrinsic::arm_mve_hadd_predicated:
19348 case Intrinsic::arm_mve_vqdmull:
19349 case Intrinsic::arm_mve_vqdmull_predicated:
19350 case Intrinsic::arm_mve_vqdmulh:
19351 case Intrinsic::arm_mve_qdmulh_predicated:
19352 case Intrinsic::arm_mve_vqrdmulh:
19353 case Intrinsic::arm_mve_qrdmulh_predicated:
19354 case Intrinsic::arm_mve_fma_predicated:
19356 case Intrinsic::ssub_sat:
19357 case Intrinsic::usub_sat:
19358 case Intrinsic::arm_mve_sub_predicated:
19359 case Intrinsic::arm_mve_qsub_predicated:
19360 case Intrinsic::arm_mve_hsub_predicated:
19361 case Intrinsic::arm_mve_vhsub:
19362 return Operand == 1;
19373 for (
auto OpIdx :
enumerate(
I->operands())) {
19374 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
19376 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
19380 if (Shuffle->
getOpcode() == Instruction::BitCast)
19381 Shuffle = dyn_cast<Instruction>(Shuffle->
getOperand(0));
19388 if (!IsSinker(
I, OpIdx.index()))
19393 for (
Use &U :
Op->uses()) {
19395 if (!IsSinker(
Insn, U.getOperandNo()))
19408 if (!Subtarget->hasMVEIntegerOps())
19426 if (
auto *Ld = dyn_cast<MaskedLoadSDNode>(ExtVal.
getOperand(0))) {
19427 if (Ld->isExpandingLoad())
19431 if (Subtarget->hasMVEIntegerOps())
19476bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(
const MachineFunction &MF,
19484 return Subtarget->hasMVEFloatOps();
19502 unsigned Scale = 1;
19519 if ((V & (Scale - 1)) != 0)
19521 return isUInt<5>(V / Scale);
19528 if (VT.
isVector() && Subtarget->hasNEON())
19531 !Subtarget->hasMVEFloatOps())
19534 bool IsNeg =
false;
19540 unsigned NumBytes = std::max((
unsigned)VT.
getSizeInBits() / 8, 1U);
19543 if (VT.
isVector() && Subtarget->hasMVEIntegerOps()) {
19547 return isShiftedUInt<7,2>(V);
19550 return isShiftedUInt<7,1>(V);
19552 return isUInt<7>(V);
19559 if (VT.
isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19560 return isShiftedUInt<8, 1>(V);
19563 return isShiftedUInt<8, 2>(V);
19565 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19568 return isUInt<8>(V);
19569 return isUInt<12>(V);
19595 default:
return false;
19600 return isUInt<12>(V);
19603 return isUInt<8>(V);
19608 return isShiftedUInt<8, 2>(V);
19614 int Scale = AM.
Scale;
19619 default:
return false;
19627 Scale = Scale & ~1;
19628 return Scale == 2 || Scale == 4 || Scale == 8;
19645 if (Scale & 1)
return false;
19652 const int Scale = AM.
Scale;
19662 return (Scale == 1) || (!AM.
HasBaseReg && Scale == 2);
19678 switch (AM.
Scale) {
19695 int Scale = AM.
Scale;
19697 default:
return false;
19701 if (Scale < 0) Scale = -Scale;
19709 if (Scale == 1 || (AM.
HasBaseReg && Scale == -1))
19722 if (Scale & 1)
return false;
19735 if (!Subtarget->isThumb())
19742 return Imm >= 0 && Imm <= 255;
19751 int64_t AbsImm = std::abs(Imm);
19752 if (!Subtarget->isThumb())
19757 return AbsImm >= 0 && AbsImm <= 255;
19792 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
19796 int RHSC = (int)
RHS->getZExtValue();
19797 if (RHSC < 0 && RHSC > -256) {
19807 }
else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
19810 int RHSC = (int)
RHS->getZExtValue();
19811 if (RHSC < 0 && RHSC > -0x1000) {
19853 int RHSC = (int)
RHS->getZExtValue();
19854 if (RHSC < 0 && RHSC > -0x100) {
19859 }
else if (RHSC > 0 && RHSC < 0x100) {
19870 bool isSEXTLoad,
bool IsMasked,
bool isLE,
19875 if (!isa<ConstantSDNode>(
Ptr->getOperand(1)))
19881 bool CanChangeType = isLE && !IsMasked;
19884 int RHSC = (int)
RHS->getZExtValue();
19886 auto IsInRange = [&](
int RHSC,
int Limit,
int Scale) {
19887 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
19892 }
else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
19903 if (VT == MVT::v4i16) {
19904 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
19906 }
else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
19907 if (IsInRange(RHSC, 0x80, 1))
19909 }
else if (Alignment >= 4 &&
19910 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
19911 IsInRange(RHSC, 0x80, 4))
19913 else if (Alignment >= 2 &&
19914 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
19915 IsInRange(RHSC, 0x80, 2))
19917 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
19936 bool isSEXTLoad =
false;
19937 bool IsMasked =
false;
19939 Ptr = LD->getBasePtr();
19940 VT = LD->getMemoryVT();
19941 Alignment = LD->getAlign();
19943 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
19944 Ptr = ST->getBasePtr();
19945 VT = ST->getMemoryVT();
19946 Alignment = ST->getAlign();
19948 Ptr = LD->getBasePtr();
19949 VT = LD->getMemoryVT();
19950 Alignment = LD->getAlign();
19954 Ptr = ST->getBasePtr();
19955 VT = ST->getMemoryVT();
19956 Alignment = ST->getAlign();
19962 bool isLegal =
false;
19964 isLegal = Subtarget->hasMVEIntegerOps() &&
19966 Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
19994 bool isSEXTLoad =
false, isNonExt;
19995 bool IsMasked =
false;
19997 VT = LD->getMemoryVT();
19998 Ptr = LD->getBasePtr();
19999 Alignment = LD->getAlign();
20002 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20003 VT = ST->getMemoryVT();
20004 Ptr = ST->getBasePtr();
20005 Alignment = ST->getAlign();
20006 isNonExt = !ST->isTruncatingStore();
20008 VT = LD->getMemoryVT();
20009 Ptr = LD->getBasePtr();
20010 Alignment = LD->getAlign();
20015 VT = ST->getMemoryVT();
20016 Ptr = ST->getBasePtr();
20017 Alignment = ST->getAlign();
20018 isNonExt = !ST->isTruncatingStore();
20026 assert(
Op->getValueType(0) == MVT::i32 &&
"Non-i32 post-inc op?!");
20027 if (
Op->getOpcode() !=
ISD::ADD || !isNonExt)
20029 auto *
RHS = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
20030 if (!
RHS ||
RHS->getZExtValue() != 4)
20032 if (Alignment <
Align(4))
20036 Base =
Op->getOperand(0);
20042 bool isLegal =
false;
20044 isLegal = Subtarget->hasMVEIntegerOps() &&
20077 const APInt &DemandedElts,
20079 unsigned Depth)
const {
20082 switch (
Op.getOpcode()) {
20089 if (
Op.getResNo() == 0) {
20115 case Intrinsic::arm_ldaex:
20116 case Intrinsic::arm_ldrex: {
20117 EVT VT = cast<MemIntrinsicSDNode>(
Op)->getMemoryVT();
20131 const APInt &Mask =
Op.getConstantOperandAPInt(2);
20132 Known.
Zero &= Mask;
20138 const SDValue &SrcSV =
Op.getOperand(0);
20144 "VGETLANE index out of bounds");
20149 EVT VT =
Op.getValueType();
20156 Known = Known.
sext(DstSz);
20158 Known = Known.
zext(DstSz);
20166 Known = KnownOp.
zext(32);
20181 true,
false,
false, KnownOp1,
20207 EVT VT =
Op.getValueType();
20213 assert(VT == MVT::i32 &&
"Unexpected integer type");
20220 unsigned Mask =
C->getZExtValue();
20223 unsigned ShrunkMask = Mask & Demanded;
20224 unsigned ExpandedMask = Mask | ~Demanded;
20228 if (ShrunkMask == 0)
20234 if (ExpandedMask == ~0U)
20237 auto IsLegalMask = [ShrunkMask, ExpandedMask](
unsigned Mask) ->
bool {
20238 return (ShrunkMask & Mask) == ShrunkMask && (~ExpandedMask & Mask) == 0;
20240 auto UseMask = [Mask,
Op, VT, &TLO](
unsigned NewMask) ->
bool {
20241 if (NewMask == Mask)
20250 if (IsLegalMask(0xFF))
20251 return UseMask(0xFF);
20254 if (IsLegalMask(0xFFFF))
20255 return UseMask(0xFFFF);
20259 if (ShrunkMask < 256)
20260 return UseMask(ShrunkMask);
20264 if ((
int)ExpandedMask <= -2 && (
int)ExpandedMask >= -256)
20265 return UseMask(ExpandedMask);
20280 unsigned Depth)
const {
20281 unsigned Opc =
Op.getOpcode();
20289 if (
Op.getResNo() == 0 && !
Op->hasAnyUseOfValue(1) &&
20290 isa<ConstantSDNode>(
Op->getOperand(2))) {
20291 unsigned ShAmt =
Op->getConstantOperandVal(2);
20303 unsigned ModImm =
Op.getConstantOperandVal(1);
20304 unsigned EltBits = 0;
20306 if ((OriginalDemandedBits & Mask) == 0)
20312 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO,
Depth);
20321 if (!Subtarget->hasV6Ops())
20327 SplitString(AsmStr, AsmPieces,
";\n");
20329 switch (AsmPieces.
size()) {
20330 default:
return false;
20332 AsmStr = AsmPieces[0];
20334 SplitString(AsmStr, AsmPieces,
" \t,");
20337 if (AsmPieces.
size() == 3 &&
20338 AsmPieces[0] ==
"rev" && AsmPieces[1] ==
"$0" && AsmPieces[2] ==
"$1" &&
20339 IA->getConstraintString().compare(0, 4,
"=l,l") == 0) {
20362 if (ConstraintVT.
isVector() && Subtarget->hasNEON() &&
20374 unsigned S = Constraint.
size();
20376 switch (Constraint[0]) {
20388 }
else if (S == 2) {
20389 switch (Constraint[0]) {
20406 Value *CallOperandVal =
info.CallOperandVal;
20409 if (!CallOperandVal)
20413 switch (*constraint) {
20419 if (Subtarget->isThumb())
20433using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20437 switch (Constraint.
size()) {
20440 switch (Constraint[0]) {
20442 if (Subtarget->isThumb())
20443 return RCPair(0U, &ARM::tGPRRegClass);
20444 return RCPair(0U, &ARM::GPRRegClass);
20446 if (Subtarget->isThumb())
20447 return RCPair(0U, &ARM::hGPRRegClass);
20451 return RCPair(0U, &ARM::tGPRRegClass);
20452 return RCPair(0U, &ARM::GPRRegClass);
20454 if (VT == MVT::Other)
20456 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20457 return RCPair(0U, &ARM::SPRRegClass);
20459 return RCPair(0U, &ARM::DPRRegClass);
20461 return RCPair(0U, &ARM::QPRRegClass);
20464 if (VT == MVT::Other)
20466 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20467 return RCPair(0U, &ARM::SPR_8RegClass);
20469 return RCPair(0U, &ARM::DPR_8RegClass);
20471 return RCPair(0U, &ARM::QPR_8RegClass);
20474 if (VT == MVT::Other)
20476 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20477 return RCPair(0U, &ARM::SPRRegClass);
20479 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20481 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20487 if (Constraint[0] ==
'T') {
20488 switch (Constraint[1]) {
20492 return RCPair(0U, &ARM::tGPREvenRegClass);
20494 return RCPair(0U, &ARM::tGPROddRegClass);
20503 if (
StringRef(
"{cc}").equals_insensitive(Constraint))
20504 return std::make_pair(
unsigned(ARM::CPSR), &ARM::CCRRegClass);
20513 std::vector<SDValue> &Ops,
20518 if (Constraint.
size() != 1)
20521 char ConstraintLetter = Constraint[0];
20522 switch (ConstraintLetter) {
20525 case 'I':
case 'J':
case 'K':
case 'L':
20526 case 'M':
case 'N':
case 'O':
20531 int64_t CVal64 =
C->getSExtValue();
20532 int CVal = (int) CVal64;
20535 if (CVal != CVal64)
20538 switch (ConstraintLetter) {
20542 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
20543 if (CVal >= 0 && CVal <= 65535)
20550 if (CVal >= 0 && CVal <= 255)
20552 }
else if (Subtarget->
isThumb2()) {
20571 if (CVal >= -255 && CVal <= -1)
20577 if (CVal >= -4095 && CVal <= 4095)
20590 }
else if (Subtarget->
isThumb2()) {
20613 if (CVal >= -7 && CVal < 7)
20615 }
else if (Subtarget->
isThumb2()) {
20638 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
20644 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
20652 if (CVal >= 0 && CVal <= 31)
20661 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
20670 if (Result.getNode()) {
20671 Ops.push_back(Result);
20681 "Unhandled Opcode in getDivRemLibcall");
20687 case MVT::i8: LC =
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
20688 case MVT::i16: LC =
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
20689 case MVT::i32: LC =
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
20690 case MVT::i64: LC =
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
20699 "Unhandled Opcode in getDivRemArgList");
20704 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
20705 EVT ArgVT =
N->getOperand(i).getValueType();
20707 Entry.Node =
N->getOperand(i);
20711 Args.push_back(Entry);
20722 "Register-based DivRem lowering only");
20723 unsigned Opcode =
Op->getOpcode();
20725 "Invalid opcode for Div/Rem lowering");
20727 EVT VT =
Op->getValueType(0);
20730 if (VT == MVT::i64 && isa<ConstantSDNode>(
Op.getOperand(1))) {
20749 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
20750 : Subtarget->hasDivideInARMMode();
20751 if (hasDivide &&
Op->getValueType(0).isSimple() &&
20752 Op->getSimpleValueType(0) == MVT::i32) {
20754 const SDValue Dividend =
Op->getOperand(0);
20755 const SDValue Divisor =
Op->getOperand(1);
20756 SDValue Div = DAG.
getNode(DivOpcode, dl, VT, Dividend, Divisor);
20760 SDValue Values[2] = {Div, Rem};
20792 EVT VT =
N->getValueType(0);
20794 if (VT == MVT::i64 && isa<ConstantSDNode>(
N->getOperand(1))) {
20798 Result[0], Result[1]);
20802 std::vector<Type*> RetTyParams;
20803 Type *RetTyElement;
20813 RetTyParams.push_back(RetTyElement);
20814 RetTyParams.push_back(RetTyElement);
20831 CallLoweringInfo CLI(DAG);
20835 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
20838 SDNode *ResNode = CallResult.first.getNode();
20853 "no-stack-arg-probe")) {
20855 cast<ConstantSDNode>(
Op.getOperand(2))->getMaybeAlignValue();
20864 SDValue Ops[2] = { SP, Chain };
20881 SDValue Ops[2] = { NewSP, Chain };
20886 bool IsStrict =
Op->isStrictFPOpcode();
20887 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20888 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20890 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
20891 "Unexpected type for custom-lowering FP_EXTEND");
20894 "With both FP DP and 16, any FP conversion is legal!");
20896 assert(!(DstSz == 32 && Subtarget->hasFP16()) &&
20897 "With FP16, 16 to 32 conversion is legal!");
20900 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) {
20905 Loc,
Op.getValueType(), SrcVal);
20918 MakeLibCallOptions CallOptions;
20920 for (
unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
20921 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
20922 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
20923 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
20927 {DstVT, MVT::Other}, {Chain, SrcVal});
20934 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20935 "Unexpected type for custom-lowering FP_EXTEND");
20936 std::tie(SrcVal, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20941 return IsStrict ? DAG.
getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
20945 bool IsStrict =
Op->isStrictFPOpcode();
20947 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20949 EVT DstVT =
Op.getValueType();
20950 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20953 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
20954 "Unexpected type for custom-lowering FP_ROUND");
20957 "With both FP DP and 16, any FP conversion is legal!");
20962 if (SrcSz == 32 && Subtarget->hasFP16())
20967 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20968 "Unexpected type for custom-lowering FP_ROUND");
20969 MakeLibCallOptions CallOptions;
20972 std::tie(Result, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20984 if (v == 0xffffffff)
20996 bool ForCodeSize)
const {
20999 if (VT == MVT::f16 && Subtarget->hasFullFP16())
21001 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
21004 if (VT == MVT::f32)
21006 if (VT == MVT::f64 && Subtarget->hasFP64())
21017 unsigned Intrinsic)
const {
21018 switch (Intrinsic) {
21019 case Intrinsic::arm_neon_vld1:
21020 case Intrinsic::arm_neon_vld2:
21021 case Intrinsic::arm_neon_vld3:
21022 case Intrinsic::arm_neon_vld4:
21023 case Intrinsic::arm_neon_vld2lane:
21024 case Intrinsic::arm_neon_vld3lane:
21025 case Intrinsic::arm_neon_vld4lane:
21026 case Intrinsic::arm_neon_vld2dup:
21027 case Intrinsic::arm_neon_vld3dup:
21028 case Intrinsic::arm_neon_vld4dup: {
21031 auto &
DL =
I.getDataLayout();
21032 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21034 Info.ptrVal =
I.getArgOperand(0);
21036 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21037 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21042 case Intrinsic::arm_neon_vld1x2:
21043 case Intrinsic::arm_neon_vld1x3:
21044 case Intrinsic::arm_neon_vld1x4: {
21047 auto &
DL =
I.getDataLayout();
21048 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21050 Info.ptrVal =
I.getArgOperand(
I.arg_size() - 1);
21052 Info.align.reset();
21057 case Intrinsic::arm_neon_vst1:
21058 case Intrinsic::arm_neon_vst2:
21059 case Intrinsic::arm_neon_vst3:
21060 case Intrinsic::arm_neon_vst4:
21061 case Intrinsic::arm_neon_vst2lane:
21062 case Intrinsic::arm_neon_vst3lane:
21063 case Intrinsic::arm_neon_vst4lane: {
21066 auto &
DL =
I.getDataLayout();
21067 unsigned NumElts = 0;
21068 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21069 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21072 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21075 Info.ptrVal =
I.getArgOperand(0);
21077 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21078 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21083 case Intrinsic::arm_neon_vst1x2:
21084 case Intrinsic::arm_neon_vst1x3:
21085 case Intrinsic::arm_neon_vst1x4: {
21088 auto &
DL =
I.getDataLayout();
21089 unsigned NumElts = 0;
21090 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21091 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21094 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21097 Info.ptrVal =
I.getArgOperand(0);
21099 Info.align.reset();
21104 case Intrinsic::arm_mve_vld2q:
21105 case Intrinsic::arm_mve_vld4q: {
21108 Type *VecTy = cast<StructType>(
I.getType())->getElementType(1);
21109 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21111 Info.ptrVal =
I.getArgOperand(0);
21118 case Intrinsic::arm_mve_vst2q:
21119 case Intrinsic::arm_mve_vst4q: {
21122 Type *VecTy =
I.getArgOperand(1)->getType();
21123 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21125 Info.ptrVal =
I.getArgOperand(0);
21132 case Intrinsic::arm_mve_vldr_gather_base:
21133 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21135 Info.ptrVal =
nullptr;
21141 case Intrinsic::arm_mve_vldr_gather_base_wb:
21142 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21144 Info.ptrVal =
nullptr;
21150 case Intrinsic::arm_mve_vldr_gather_offset:
21151 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21153 Info.ptrVal =
nullptr;
21155 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
21162 case Intrinsic::arm_mve_vstr_scatter_base:
21163 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21165 Info.ptrVal =
nullptr;
21171 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21172 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21174 Info.ptrVal =
nullptr;
21180 case Intrinsic::arm_mve_vstr_scatter_offset:
21181 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21183 Info.ptrVal =
nullptr;
21185 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(3))->getZExtValue();
21192 case Intrinsic::arm_ldaex:
21193 case Intrinsic::arm_ldrex: {
21194 auto &
DL =
I.getDataLayout();
21195 Type *ValTy =
I.getParamElementType(0);
21198 Info.ptrVal =
I.getArgOperand(0);
21200 Info.align =
DL.getABITypeAlign(ValTy);
21204 case Intrinsic::arm_stlex:
21205 case Intrinsic::arm_strex: {
21206 auto &
DL =
I.getDataLayout();
21207 Type *ValTy =
I.getParamElementType(1);
21210 Info.ptrVal =
I.getArgOperand(1);
21212 Info.align =
DL.getABITypeAlign(ValTy);
21216 case Intrinsic::arm_stlexd:
21217 case Intrinsic::arm_strexd:
21219 Info.memVT = MVT::i64;
21220 Info.ptrVal =
I.getArgOperand(2);
21226 case Intrinsic::arm_ldaexd:
21227 case Intrinsic::arm_ldrexd:
21229 Info.memVT = MVT::i64;
21230 Info.ptrVal =
I.getArgOperand(0);
21250 if (Bits == 0 || Bits > 32)
21256 unsigned Index)
const {
21268 if (!Subtarget->hasDataBarrier()) {
21272 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21309 if (Subtarget->preferISHSTBarriers())
21342 bool has64BitAtomicStore;
21344 has64BitAtomicStore =
false;
21345 else if (Subtarget->isThumb())
21346 has64BitAtomicStore = Subtarget->hasV7Ops();
21348 has64BitAtomicStore = Subtarget->hasV6Ops();
21350 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21364 bool has64BitAtomicLoad;
21366 has64BitAtomicLoad =
false;
21367 else if (Subtarget->isThumb())
21368 has64BitAtomicLoad = Subtarget->hasV7Ops();
21370 has64BitAtomicLoad = Subtarget->hasV6Ops();
21387 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21388 else if (Subtarget->isThumb())
21389 hasAtomicRMW = Subtarget->hasV7Ops();
21391 hasAtomicRMW = Subtarget->hasV6Ops();
21392 if (
Size <= (Subtarget->
isMClass() ? 32U : 64U) && hasAtomicRMW) {
21415 bool HasAtomicCmpXchg;
21417 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21418 else if (Subtarget->isThumb())
21419 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21421 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21423 HasAtomicCmpXchg &&
Size <= (Subtarget->
isMClass() ? 32U : 64U))
21430 return InsertFencesForAtomic;
21443 M.getOrInsertGlobal(
"__security_cookie",
21451 F->addParamAttr(0, Attribute::AttrKind::InReg);
21457 return M.getGlobalVariable(
"__security_cookie");
21464 return M.getFunction(
"__security_check_cookie");
21469 unsigned &
Cost)
const {
21471 if (!Subtarget->hasNEON())
21483 if (!isa<ConstantInt>(
Idx))
21498 return Subtarget->hasV6T2Ops();
21502 return Subtarget->hasV6T2Ops();
21507 if (!Subtarget->hasV7Ops())
21513 if (!Mask || Mask->getValue().getBitWidth() > 32u)
21515 auto MaskVal =
unsigned(Mask->getValue().getZExtValue());
21540 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
21552 Lo, Builder.
CreateShl(
Hi, ConstantInt::get(ValueTy, 32)),
"val64");
21555 Type *Tys[] = {
Addr->getType() };
21556 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
21561 0,
Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
21567 if (!Subtarget->hasV7Ops())
21584 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
21595 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
21596 Type *Tys[] = {
Addr->getType() };
21618 return (
DL.getTypeSizeInBits(VecTy) + 127) / 128;
21625 unsigned VecSize =
DL.getTypeSizeInBits(VecTy);
21628 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
21636 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
21644 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
21647 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
21652 if (Subtarget->hasNEON() && VecSize == 64)
21654 return VecSize % 128 == 0;
21658 if (Subtarget->hasNEON())
21660 if (Subtarget->hasMVEIntegerOps())
21680 "Invalid interleave factor");
21681 assert(!Shuffles.
empty() &&
"Empty shufflevector input");
21683 "Unmatched number of shufflevectors and indices");
21685 auto *VecTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21686 Type *EltTy = VecTy->getElementType();
21709 if (NumLoads > 1) {
21713 VecTy->getNumElements() / NumLoads);
21718 auto createLoadIntrinsic = [&](
Value *BaseAddr) {
21719 if (Subtarget->hasNEON()) {
21721 Type *Tys[] = {VecTy, PtrTy};
21722 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
21723 Intrinsic::arm_neon_vld3,
21724 Intrinsic::arm_neon_vld4};
21732 return Builder.
CreateCall(VldnFunc, Ops,
"vldN");
21734 assert((Factor == 2 || Factor == 4) &&
21735 "expected interleave factor of 2 or 4 for MVE");
21737 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
21739 Type *Tys[] = {VecTy, PtrTy};
21745 return Builder.
CreateCall(VldnFunc, Ops,
"vldN");
21754 for (
unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
21759 VecTy->getNumElements() * Factor);
21761 CallInst *VldN = createLoadIntrinsic(BaseAddr);
21765 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21767 unsigned Index = Indices[i];
21777 SubVecs[SV].push_back(SubVec);
21786 auto &SubVec = SubVecs[SVI];
21789 SVI->replaceAllUsesWith(WideVec);
21823 unsigned Factor)
const {
21825 "Invalid interleave factor");
21827 auto *VecTy = cast<FixedVectorType>(SVI->
getType());
21828 assert(VecTy->getNumElements() % Factor == 0 &&
"Invalid interleaved store");
21830 unsigned LaneLen = VecTy->getNumElements() / Factor;
21831 Type *EltTy = VecTy->getElementType();
21835 Align Alignment = SI->getAlign();
21852 Type *IntTy =
DL.getIntPtrType(EltTy);
21864 Value *BaseAddr = SI->getPointerOperand();
21866 if (NumStores > 1) {
21869 LaneLen /= NumStores;
21877 auto createStoreIntrinsic = [&](
Value *BaseAddr,
21879 if (Subtarget->hasNEON()) {
21880 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
21881 Intrinsic::arm_neon_vst3,
21882 Intrinsic::arm_neon_vst4};
21883 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21884 Type *Tys[] = {PtrTy, SubVecTy};
21887 SI->getModule(), StoreInts[Factor - 2], Tys);
21895 assert((Factor == 2 || Factor == 4) &&
21896 "expected interleave factor of 2 or 4 for MVE");
21898 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
21899 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21900 Type *Tys[] = {PtrTy, SubVecTy};
21907 for (
unsigned F = 0;
F < Factor;
F++) {
21915 for (
unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
21918 if (StoreCount > 0)
21920 BaseAddr, LaneLen * Factor);
21925 for (
unsigned i = 0; i < Factor; i++) {
21926 unsigned IdxI = StoreCount * LaneLen * Factor + i;
21927 if (Mask[IdxI] >= 0) {
21931 unsigned StartMask = 0;
21932 for (
unsigned j = 1; j < LaneLen; j++) {
21933 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
21934 if (Mask[IdxJ * Factor + IdxI] >= 0) {
21935 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
21950 createStoreIntrinsic(BaseAddr, Shuffles);
21965 if (
auto *ST = dyn_cast<StructType>(Ty)) {
21966 for (
unsigned i = 0; i < ST->getNumElements(); ++i) {
21970 Members += SubMembers;
21972 }
else if (
auto *AT = dyn_cast<ArrayType>(Ty)) {
21976 Members += SubMembers * AT->getNumElements();
21987 }
else if (
auto *VT = dyn_cast<VectorType>(Ty)) {
21994 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
21996 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
21998 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
22011 return (Members > 0 && Members <= 4);
22017 const Align ABITypeAlign =
DL.getABITypeAlign(ArgTy);
22019 return ABITypeAlign;
22023 return std::min(ABITypeAlign,
DL.getStackAlignment());
22032 if (getEffectiveCallingConv(CallConv, isVarArg) !=
22042 return IsHA || IsIntArray;
22046 const Constant *PersonalityFn)
const {
22053 const Constant *PersonalityFn)
const {
22065void ARMTargetLowering::insertCopiesSplitCSR(
22069 const MCPhysReg *IStart =
TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
22079 RC = &ARM::GPRRegClass;
22080 else if (ARM::DPRRegClass.
contains(*
I))
22081 RC = &ARM::DPRRegClass;
22091 assert(Entry->getParent()->getFunction().hasFnAttribute(
22092 Attribute::NoUnwind) &&
22093 "Function should be nounwind in insertCopiesSplitCSR!");
22094 Entry->addLiveIn(*
I);
22099 for (
auto *Exit : Exits)
22101 TII->get(TargetOpcode::COPY), *
I)
22112 return Subtarget->hasMVEIntegerOps();
22117 auto *VTy = dyn_cast<FixedVectorType>(Ty);
22122 unsigned NumElements = VTy->getNumElements();
22129 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22130 return Subtarget->hasMVEFloatOps();
22135 return Subtarget->hasMVEIntegerOps() &&
22136 (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
22137 ScalarTy->isIntegerTy(32));
22149 assert(TyWidth >= 128 &&
"Width of vector type must be at least 128 bits");
22151 if (TyWidth > 128) {
22156 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22158 auto *LowerSplitA =
B.CreateShuffleVector(InputA, LowerSplitMask);
22159 auto *LowerSplitB =
B.CreateShuffleVector(InputB, LowerSplitMask);
22160 auto *UpperSplitA =
B.CreateShuffleVector(InputA, UpperSplitMask);
22161 auto *UpperSplitB =
B.CreateShuffleVector(InputB, UpperSplitMask);
22162 Value *LowerSplitAcc =
nullptr;
22163 Value *UpperSplitAcc =
nullptr;
22166 LowerSplitAcc =
B.CreateShuffleVector(
Accumulator, LowerSplitMask);
22167 UpperSplitAcc =
B.CreateShuffleVector(
Accumulator, UpperSplitMask);
22171 B, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
22173 B, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);
22176 return B.CreateShuffleVector(LowerSplitInt, UpperSplitInt, JoinMask);
22183 ConstRotation = ConstantInt::get(IntTy, (
int)Rotation);
22186 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmlaq, Ty,
22188 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmulq, Ty,
22189 {ConstRotation, InputB, InputA});
22194 auto *ConstHalving = ConstantInt::get(IntTy, 1);
22197 ConstRotation = ConstantInt::get(IntTy, 0);
22199 ConstRotation = ConstantInt::get(IntTy, 1);
22201 if (!ConstRotation)
22204 return B.CreateIntrinsic(Intrinsic::arm_mve_vcaddq, Ty,
22205 {ConstHalving, ConstRotation, InputA, InputB});
unsigned const MachineRegisterInfo * MRI
static bool isAddSubSExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG, bool isSigned)
static bool isZeroExtended(SDValue N, SelectionDAG &DAG)
static bool areExtractExts(Value *Ext1, Value *Ext2)
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static const MCPhysReg GPRArgRegs[]
static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1, SDValue V2, unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
static bool isSignExtended(SDValue N, SelectionDAG &DAG)
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
static bool isAddSubZExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG)
static bool isStore(int Opcode)
static bool isThumb(const MCSubtargetInfo &STI)
static SDValue LowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, const TargetInstrInfo *TII)
MatchingStackOffset - Return true if the given stack call argument is already available in the same p...
static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG)
static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total value size to 64 bits.
static cl::opt< unsigned > ConstpoolPromotionMaxSize("arm-promote-constant-max-size", cl::Hidden, cl::desc("Maximum size of constant to promote into a constant pool"), cl::init(64))
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isVTBLMask(ArrayRef< int > M, EVT VT)
static SDValue PerformSUBCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
static cl::opt< bool > EnableConstpoolPromotion("arm-promote-constant", cl::Hidden, cl::desc("Enable / disable promotion of unnamed_addr constants into " "constant pools"), cl::init(false))
static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG)
static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
static SDValue PerformExtractEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static const APInt * isPowerOf2Constant(SDValue V)
static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG)
static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static bool isValidMVECond(unsigned CC, bool IsFloat)
static SDValue PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC)
IntCCToARMCC - Convert a DAG integer condition code to an ARM CC.
static SDValue PerformSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSTORECombine - Target-specific dag combine xforms for ISD::STORE.
static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry, SelectionDAG &DAG)
static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isGTorGE(ISD::CondCode CC)
static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a vldN-lane (N > 1) intrinsic,...
static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask)
static bool isReverseMask(ArrayRef< int > M, EVT VT)
static bool isVZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of "vector_shuffle v,...
static SDValue PerformSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVMulVCTPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVMulVCTPCombine - VCVT (fixed-point to floating-point, Advanced SIMD) can replace combinations...
static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG)
static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc)
static bool isVTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool CanInvertMVEVCMP(SDValue N)
static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG)
static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformShiftCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
PerformShiftCombine - Checks for immediate versions of vector shifts and lowers them.
static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, ARMCC::CondCodes &CondCode2)
FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static EVT getVectorTyFromPredicateVector(EVT VT)
static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG)
static SDValue handleCMSEValue(const SDValue &Value, const ISD::InputArg &Arg, SelectionDAG &DAG, const SDLoc &DL)
static SDValue PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
static bool isSRL16(const SDValue &Op)
static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC)
static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr, SDValue Inc, const SelectionDAG &DAG)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static Register genTPEntry(MachineBasicBlock *TpEntry, MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpExit, Register OpSizeReg, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI)
Adds logic in loop entry MBB to calculate loop iteration count and adds t2WhileLoopSetup and t2WhileL...
static bool isLTorLE(ISD::CondCode CC)
static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl, SelectionDAG &DAG)
static SDValue PerformBITCASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG)
static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG)
static bool hasNormalLoadOperand(SDNode *N)
hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node are normal,...
static SDValue PerformInsertEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
PerformInsertEltCombine - Target-specific dag combine xforms for ISD::INSERT_VECTOR_ELT.
static SDValue PerformVDUPLANECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVDUPLANECombine - Target-specific dag combine xforms for ARMISD::VDUPLANE.
static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static cl::opt< unsigned > ConstpoolPromotionMaxTotal("arm-promote-constant-max-total", cl::Hidden, cl::desc("Maximum size of ALL constants to promote into a constant pool"), cl::init(128))
static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static RTLIB::Libcall getDivRemLibcall(const SDNode *N, MVT::SimpleValueType SVT)
static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG &DAG)
SkipLoadExtensionForVMULL - return a load of the original vector size that does not do any sign/zero ...
static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static const MCPhysReg GPRArgRegs[]
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombineWithOperands - Try DAG combinations for an ADD with operands N0 and N1.
static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT, SelectionDAG &DAG)
static bool isVZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformORCombineToSMULWBT(SDNode *OR, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isVTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of "vector_shuffle v,...
static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue FindBFIToCombineWith(SDNode *N)
static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT, SelectionDAG &DAG)
static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, bool &swpCmpOps, bool &swpVselOps)
static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isS16(const SDValue &Op, SelectionDAG &DAG)
static bool isSRA16(const SDValue &Op)
static SDValue AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue LowerInterruptReturn(SmallVectorImpl< SDValue > &RetOps, const SDLoc &DL, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl, SelectionDAG &DAG)
static void expandf64Toi32(SDValue Op, SelectionDAG &DAG, SDValue &RetVal1, SDValue &RetVal2)
static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isSHL16(const SDValue &Op)
static bool isVEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseVEXT, unsigned &Imm)
static SDValue PerformMVEVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isTruncMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue PerformADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2)
Return the load opcode for a given load size.
static bool isLegalT2AddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
static bool isLegalMVEShuffleOp(unsigned PFEntry)
static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N, SelectionDAG &DAG)
static bool isVUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG)
PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for ISD::VECTOR_SHUFFLE.
static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG)
SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND, ANY_EXTEND,...
static bool isVMOVNTruncMask(ArrayRef< int > M, EVT ToVT, bool rev)
static SDValue PerformVQMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static MachineBasicBlock * OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ)
static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformAddcSubcCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static TargetLowering::ArgListTy getDivRemArgList(const SDNode *N, LLVMContext *Context, const ARMSubtarget *Subtarget)
static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl)
getZeroVector - Returns a vector of specified type with all zero elements.
static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static ARMCC::CondCodes getVCMPCondCode(SDValue N)
static cl::opt< bool > ARMInterworking("arm-interworking", cl::Hidden, cl::desc("Enable / disable ARM interworking (for debugging only)"), cl::init(true))
static void ReplaceREADCYCLECOUNTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformORCombineToBFI(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, SDValue &CC, bool &Invert, SDValue &OtherOp, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVSetCCToVCTPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isZeroVector(SDValue N)
static SDValue PerformAddeSubeCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void ReplaceCMP_SWAP_64Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isLowerSaturate(const SDValue LHS, const SDValue RHS, const SDValue TrueVal, const SDValue FalseVal, const ISD::CondCode CC, const SDValue K)
static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG)
static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned StSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment store operation with given size.
static bool isVMOVNMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue CombineBaseUpdate(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineBaseUpdate - Target-specific DAG combine function for VLDDUP, NEON load/store intrinsics,...
static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVRRDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMOVRRDCombine - Target-specific dag combine xforms for ARMISD::VMOVRRD.
static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain)
static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue PerformVMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMULCombine Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the special multi...
static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG)
static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformORCombine - Target-specific dag combine xforms for ISD::OR.
static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG)
static SDValue PerformTruncatingStoreCombine(StoreSDNode *St, SelectionDAG &DAG)
static unsigned SelectPairHalf(unsigned Elements, ArrayRef< int > Mask, unsigned Index)
static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned LdSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment load operation with given size.
static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG)
static bool isValidBaseUpdate(SDNode *N, SDNode *User)
static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG, const ARMSubtarget *ST, const SDLoc &dl)
static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op)
static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformXORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment, bool isSEXTLoad, bool IsMasked, bool isLE, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
std::pair< unsigned, const TargetRegisterClass * > RCPair
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,...
static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
cl::opt< unsigned > MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden, cl::desc("Maximum interleave factor for MVE VLDn to generate."), cl::init(2))
static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, unsigned SplatBitSize, SelectionDAG &DAG, const SDLoc &dl, EVT &VT, EVT VectorVT, VMOVModImmType type)
isVMOVModifiedImm - Check if the specified splat value corresponds to a valid vector constant for a N...
static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC, SelectionDAG &DAG)
BC is a bitcast that is about to be turned into a VMOVDRR.
static SDValue promoteToConstantPool(const ARMTargetLowering *TLI, const GlobalValue *GV, SelectionDAG &DAG, EVT PtrVT, const SDLoc &dl)
static unsigned isNEONTwoResultShuffleMask(ArrayRef< int > ShuffleMask, EVT VT, unsigned &WhichResult, bool &isV_UNDEF)
Check if ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN), and return the corresponding AR...
static bool BitsProperlyConcatenate(const APInt &A, const APInt &B)
static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG)
static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target, struct BaseUpdateUser &User, bool SimpleConstIncOnly, TargetLowering::DAGCombinerInfo &DCI)
static bool allUsersAreInFunction(const Value *V, const Function *F)
Return true if all users of V are within function F, looking through ConstantExprs.
static bool isSingletonVEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG)
PerformVMOVDRRCombine - Target-specific dag combine xforms for ARMISD::VMOVDRR.
static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, SDValue &SatK)
static bool isLegalAddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
isLegalAddressImmediate - Return true if the integer value can be used as the offset of the target ad...
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isLegalT1AddressImmediate(int64_t V, EVT VT)
static SDValue CombineANDShift(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSHLSimplify(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformADDECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDECombine - Target-specific dag combine transform from ARMISD::ADDC, ARMISD::ADDE,...
static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformHWLoopCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool isVUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of "vector_shuffle v,...
static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base, uint64_t &Members)
static SDValue PerformMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
static SDValue PerformANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG)
static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm, bool &Negate)
static bool canChangeToInt(SDValue Op, bool &SeenZero, const ARMSubtarget *Subtarget)
canChangeToInt - Given the fp compare operand, return true if it is suitable to morph to an integer c...
static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2)
Return the store opcode for a given store size.
static bool IsVUZPShuffleNode(SDNode *N)
static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget, MachineInstr &MI, const SDNode *Node)
Attaches vregs to MEMCPY that it will use as scratch registers when it is expanded into LDM/STM.
static bool isFloatingPointZero(SDValue Op)
isFloatingPointZero - Return true if this is +0.0.
static SDValue findMUL_LOHI(SDValue V)
static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N, SelectionDAG &DAG)
static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG)
static void genTPLoopBody(MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI, Register OpSrcReg, Register OpDestReg, Register ElementCountReg, Register TotalIterationsReg, bool IsMemcpy)
Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and t2DoLoopEnd.
static SDValue PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformBUILD_VECTORCombine - Target-specific dag combine xforms for ISD::BUILD_VECTOR.
static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformMinMaxCombine - Target-specific DAG combining for creating truncating saturates.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static Function * getFunction(Constant *C)
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
static Value * LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctpop of V before the specified instruction IP.
loop Loop Strength Reduction
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
This file describes how to lower LLVM code to machine code.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Assuming the flags are set by MI(a,b), return the condition code if we modify the instructions such t...
static constexpr int Concat[]
bool getExactInverse(APFloat *inv) const
APInt bitcastToAPInt() const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
virtual const ARMBaseRegisterInfo & getRegisterInfo() const =0
const uint32_t * getSjLjDispatchPreservedMask(const MachineFunction &MF) const
const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const override
Code Generation virtual methods...
Register getFrameRegister(const MachineFunction &MF) const override
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
const uint32_t * getTLSCallPreservedMask(const MachineFunction &MF) const
const uint32_t * getThisReturnPreservedMask(const MachineFunction &MF, CallingConv::ID) const
getThisReturnPreservedMask - Returns a call preserved mask specific to the case that 'returned' is on...
static ARMConstantPoolConstant * Create(const Constant *C, unsigned ID)
static ARMConstantPoolMBB * Create(LLVMContext &C, const MachineBasicBlock *mbb, unsigned ID, unsigned char PCAdj)
static ARMConstantPoolSymbol * Create(LLVMContext &C, StringRef s, unsigned ID, unsigned char PCAdj)
ARMConstantPoolValue - ARM specific constantpool value.
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
int getVarArgsFrameIndex() const
int getPromotedConstpoolIncrease() const
SmallPtrSet< const GlobalVariable *, 2 > & getGlobalsPromotedToConstantPool()
void setArgumentStackToRestore(unsigned v)
bool branchTargetEnforcement() const
unsigned createPICLabelUId()
void setPromotedConstpoolIncrease(int Sz)
bool isThumb1OnlyFunction() const
void setArgRegsSaveSize(unsigned s)
bool shouldSignReturnAddress() const
bool isCmseNSEntryFunction() const
void setReturnRegsCount(unsigned s)
void setVarArgsFrameIndex(int Index)
unsigned getArgRegsSaveSize() const
void markGlobalAsPromotedToConstantPool(const GlobalVariable *GV)
Indicate to the backend that GV has had its storage changed to inside a constant pool.
void setIsSplitCSR(bool s)
void setArgumentStackSize(unsigned size)
unsigned getArgumentStackSize() const
bool isTargetMachO() const
bool isTargetAEABI() const
bool supportsTailCall() const
const Triple & getTargetTriple() const
const ARMBaseInstrInfo * getInstrInfo() const override
bool isThumb1Only() const
bool hasFPARMv8Base() const
bool isTargetWindows() const
bool isGVIndirectSymbol(const GlobalValue *GV) const
True if the GV will be accessed via an indirect symbol.
const ARMTargetLowering * getTargetLowering() const override
bool isTargetDarwin() const
const ARMBaseRegisterInfo * getRegisterInfo() const override
bool isTargetAndroid() const
bool isTargetCOFF() const
bool isTargetGNUAEABI() const
bool isTargetWatchOS() const
bool useNEONForSinglePrecisionFP() const
const InstrItineraryData * getInstrItineraryData() const override
getInstrItins - Return the instruction itineraries based on subtarget selection.
bool isTargetWatchABI() const
bool hasAnyDataBarrier() const
bool isTargetDriverKit() const
bool allowsUnalignedMem() const
bool isTargetMuslAEABI() const
bool isTargetLinux() const
unsigned getPrefLoopLogAlignment() const
bool isTargetHardFloat() const
Align getDualLoadStoreAlignment() const
bool isReadOnly(const GlobalValue *GV) const
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
bool shouldInsertFencesForAtomic(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const override
Return the correct alignment for the current calling convention.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
isLegalAddressingMode - Return true if the addressing mode represented by AM is legal for this target...
const ARMSubtarget * getSubtarget() const
bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const
bool isLegalT1ScaledAddressingMode(const AddrMode &AM, EVT VT) const
Returns true if the addressing mode representing by AM is legal for the Thumb1 target,...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPreIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mod...
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
ReplaceNodeResults - Replace the results of node with an illegal result type with new values built ou...
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
bool isLegalAddImmediate(int64_t Imm) const override
isLegalAddImmediate - Return true if the specified immediate is legal add immediate,...
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
SDValue PerformMVETruncCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const override
isFPImmLegal - Returns true if the target can instruction select the specified FP immediate natively.
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
SDValue PerformIntrinsicCombine(SDNode *N, DAGCombinerInfo &DCI) const
PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
bool isDesirableToCommuteXorWithShift(const SDNode *N) const override
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
bool ExpandInlineAsm(CallInst *CI) const override
This hook allows the target to expand an inline asm call to be explicit llvm code if it wants to.
SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const
PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const override
Create the IR node for the given complex deinterleaving operation.
bool isComplexDeinterleavingSupported() const override
Does this target support complex deinterleaving.
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
SDValue PerformMVEExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
getSetCCResultType - Return the value type to use for ISD::SETCC.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool isVarArg) const
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
createFastISel - This method returns a target specific FastISel object, or null if the target does no...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
isShuffleMaskLegal - Targets can use this to indicate that they only support some VECTOR_SHUFFLE oper...
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const override
getRegClassFor - Return the register class that should be used for the specified value type.
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vstN intrinsic.
ARMTargetLowering(const TargetMachine &TM, const ARMSubtarget &STI)
bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const override
Does this target support complex deinterleaving with the given operation and type.
SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const
PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const override
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
Instruction * makeDMB(IRBuilderBase &Builder, ARM_MB::MemBOpt Domain) const
bool isLegalICmpImmediate(int64_t Imm) const override
isLegalICmpImmediate - Return true if the specified immediate is legal icmp immediate,...
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool isVarArg) const
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
allowsMisalignedMemoryAccesses - Returns true if the target allows unaligned memory accesses of the s...
bool isLegalInterleavedAccessType(unsigned Factor, FixedVectorType *VecTy, Align Alignment, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
bool isVectorLoadExtDesirable(SDValue ExtVal) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const override
Return true if the target can combine store(extractelement VectorTy, Idx).
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vldN intrinsic.
bool useSoftFloat() const override
bool alignLoopsWithOptSize() const override
Should loops be aligned even when the function is marked OptSize (but not MinSize).
SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
LowerAsmOperandForConstraint - Lower the specified operand into the Ops vector.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const override
Returns true if an argument of type Ty needs to be passed in a contiguous block of registers in calli...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPostIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mo...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
bool isFloatingPointOperation() const
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
The address of a basic block.
static BranchProbability getZero()
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
CCState - This class holds information needed while lowering arguments and return values.
void getInRegsParamInfo(unsigned InRegsParamRecordIndex, unsigned &BeginReg, unsigned &EndReg) const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
void rewindByValRegsInfo()
unsigned getInRegsParamsProcessed() const
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void addInRegsParamInfo(unsigned RegBegin, unsigned RegEnd)
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
unsigned getInRegsParamsCount() const
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
unsigned getValNo() const
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
Value * getCalledOperand() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
const APFloat & getValueAPF() const
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
Align getStackAlignment() const
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPreferredAlign(const GlobalVariable *GV) const
Returns the preferred alignment of the specified global.
StringRef getPrivateGlobalPrefix() const
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for unsupported feature in backend.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
const GlobalValue * getGlobal() const
bool hasExternalWeakLinkage() const
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
bool isStrongDefinitionForLinker() const
Returns true if this global's definition will be the one chosen by the linker.
@ InternalLinkage
Rename collisions when linking (static functions).
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
TargetInstrInfo overrides.
Common base class shared among various IRBuilders.
Value * CreateZExtOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
BasicBlock * GetInsertBlock() const
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateTruncOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::optional< unsigned > getOperandCycle(unsigned ItinClassIndx, unsigned OperandIdx) const
Return the cycle for the given class and operand.
bool isEmpty() const
Returns true if there are no itineraries.
bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
static bool LowerToByteSwap(CallInst *CI)
Try to replace a call instruction with a call to a bswap intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Describe properties that are true of each instruction in the target description file.
unsigned getSchedClass() const
Return the scheduling class for this instruction.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
bool isEHPad() const
Returns true if the block is a landing pad.
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool canFallThrough()
Return true if the block can implicitly transfer control to the block after it by falling off the end...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
pred_iterator pred_begin()
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void moveAfter(MachineBasicBlock *NewBefore)
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void computeMaxCallFrameSize(MachineFunction &MF, std::vector< MachineBasicBlock::iterator > *FrameSDOps=nullptr)
Computes the maximum size of a callframe.
void setAdjustsStack(bool V)
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool hasVAStart() const
Returns true if the function calls the llvm.va_start intrinsic.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
int getFunctionContextIndex() const
Return the index for the function context object.
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & reset(Property P)
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i....
Representation of each machine instruction.
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
const MachineOperand & getOperand(unsigned i) const
unsigned createJumpTableIndex(const std::vector< MachineBasicBlock * > &DestBBs)
createJumpTableIndex - Create a new jump table.
@ EK_Inline
EK_Inline - Jump table entries are emitted inline at their point of use.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
void setIsDef(bool Val=true)
Change a def to a use, or a use to a def.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class is used to represent an MLOAD node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
virtual void print(raw_ostream &OS, const Module *M) const
print - Print out the internal state of the pass.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< use_iterator > uses()
size_t use_size() const
Return the number of uses of this node.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
void setFlags(SDNodeFlags NewFlags)
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
void addCallSiteInfo(const SDNode *Node, CallSiteInfo &&CallInfo)
Set CallSiteInfo to be associated with Node.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
StringRef - Represent a constant reference to a string, i.e.
const unsigned char * bytes_end() const
constexpr size_t size() const
size - Get the string size.
const unsigned char * bytes_begin() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
void setCmpLibcallCC(RTLIB::Libcall Call, ISD::CondCode CC)
Override the default CondCode to be used to test the result of the comparison libcall against zero.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
void setMinStackArgumentAlignment(Align Alignment)
Set the minimum stack alignment of an argument.
const TargetMachine & getTargetMachine() const
void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
virtual unsigned getMaxSupportedInterleaveFactor() const
Get the maximum supported factor for interleaved memory accesses.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
bool isPositionIndependent() const
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
bool isOSVersionLT(unsigned Major, unsigned Minor=0, unsigned Micro=0) const
Helper function for doing comparisons against version numbers included in the target triple.
bool isWindowsMSVCEnvironment() const
Checks if the environment could be MSVC.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isArrayTy() const
True if this is an instance of ArrayType.
bool isPointerTy() const
True if this is an instance of PointerType.
Type * getArrayElementType() const
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static CondCodes getOppositeCondition(CondCodes CC)
@ SECREL
Thread Pointer Offset.
@ SBREL
Section Relative (Windows TLS)
@ GOTTPOFF
Global Offset Table, PC Relative.
@ TPOFF
Global Offset Table, Thread Pointer Offset.
TOF
Target Operand Flag enum.
@ MO_NONLAZY
MO_NONLAZY - This is an independent flag, on a symbol operand "FOO" it represents a symbol which,...
@ MO_SBREL
MO_SBREL - On a symbol operand, this represents a static base relative relocation.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_GOT
MO_GOT - On a symbol operand, this represents a GOT relative relocation.
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
unsigned getAM2Offset(unsigned AM2Opc)
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned createVMOVModImm(unsigned OpCmode, unsigned Val)
int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
int getFP32FP16Imm(const APInt &Imm)
If this is a FP16Imm encoded as a fp32 value, return the 8-bit encoding for it.
AddrOpc getAM2Op(unsigned AM2Opc)
bool isBitFieldInvertedMask(unsigned v)
const unsigned FPStatusBits
const unsigned FPReservedBits
const unsigned RoundingBitsPos
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Swift
Calling convention for Swift.
@ ARM_APCS
ARM Procedure Calling Standard (obsolete, but still used on some targets).
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ ARM_AAPCS
ARM Architecture Procedure Calling Standard calling convention (aka EABI).
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool CC_ARM_APCS_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool RetCC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns true if Val1 has a lower Constant Materialization Cost than Val2.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool FastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool RetCC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
void shuffle(Iterator first, Iterator last, RNG &&g)
static std::array< MachineOperand, 2 > predOps(ARMCC::CondCodes Pred, unsigned PredReg=0)
Get the operands corresponding to the given Pred value.
bool CC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
bool CC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_Win32_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
ComplexDeinterleavingOperation
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
ComplexDeinterleavingRotation
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
static MachineOperand t1CondCodeOp(bool isDead=false)
Get the operand corresponding to the conditional code result for Thumb1.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
static MachineOperand condCodeOp(unsigned CCReg=0)
Get the operand corresponding to the conditional code result.
bool isVREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isVREVMask - Check if a vector shuffle corresponds to a VREV instruction with the specified blocksize...
unsigned gettBLXrOpcode(const MachineFunction &MF)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
unsigned convertAddSubFlagsOpcode(unsigned OldOpc)
Map pseudo instructions that imply an 'S' bit onto real opcodes.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
static const unsigned PerfectShuffleTable[6561+1]
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Load/store instruction that can be merged with a base address update.
SDNode * N
Instruction that updates a pointer.
unsigned ConstInc
Pointer increment value if it is a constant, or 0 otherwise.
SDValue Inc
Pointer increment operand.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
static constexpr roundingMode rmTowardZero
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
bool isUnknown() const
Returns true if we don't know any bits.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setInRegister(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)