84#include "llvm/IR/IntrinsicsARM.h"
122#define DEBUG_TYPE "arm-isel"
125STATISTIC(NumMovwMovt,
"Number of GAs materialized with movw + movt");
126STATISTIC(NumLoopByVals,
"Number of loops generated for byval arguments");
128 "Number of constants with their storage promoted into constant pools");
132 cl::desc(
"Enable / disable ARM interworking (for debugging only)"),
137 cl::desc(
"Enable / disable promotion of unnamed_addr constants into "
142 cl::desc(
"Maximum size of constant to promote into a constant pool"),
146 cl::desc(
"Maximum size of ALL constants to promote into a constant pool"),
151 cl::desc(
"Maximum interleave factor for MVE VLDn to generate."),
156 ARM::R0, ARM::R1, ARM::R2, ARM::R3
170void ARMTargetLowering::addTypeForNEON(
MVT VT,
MVT PromotedLdStVT) {
171 if (VT != PromotedLdStVT) {
180 if (ElemTy != MVT::f64)
184 if (ElemTy == MVT::i32) {
228void ARMTargetLowering::addDRTypeForNEON(
MVT VT) {
230 addTypeForNEON(VT, MVT::f64);
233void ARMTargetLowering::addQRTypeForNEON(
MVT VT) {
235 addTypeForNEON(VT, MVT::v2f64);
238void ARMTargetLowering::setAllExpand(
MVT VT) {
251void ARMTargetLowering::addAllExtLoads(
const MVT From,
const MVT To,
252 LegalizeAction Action) {
258void ARMTargetLowering::addMVEVectorTypes(
bool HasMVEFP) {
259 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
261 for (
auto VT : IntTypes) {
335 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
336 for (
auto VT : FloatTypes) {
405 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
406 for (
auto VT : LongTypes) {
423 addAllExtLoads(MVT::v8i16, MVT::v8i8,
Legal);
424 addAllExtLoads(MVT::v4i32, MVT::v4i16,
Legal);
425 addAllExtLoads(MVT::v4i32, MVT::v4i8,
Legal);
442 for (
auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
451 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
452 for (
auto VT : pTypes) {
508 for (
int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID)
516 if (Subtarget->isThumb() && Subtarget->
hasVFP2Base() &&
517 Subtarget->
hasARMOps() && !Subtarget->useSoftFloat()) {
518 static const struct {
520 const char *
const Name;
542 { RTLIB::UO_F32,
"__unordsf2vfp",
ISD::SETNE },
551 { RTLIB::UO_F64,
"__unorddf2vfp",
ISD::SETNE },
576 for (
const auto &LC : LibraryCalls) {
588 static const struct {
590 const char *
const Name;
675 for (
const auto &LC : LibraryCalls) {
685 static const struct {
687 const char *
const Name;
690 } MemOpsLibraryCalls[] = {
698 for (
const auto &LC : MemOpsLibraryCalls) {
708 static const struct {
710 const char *
const Name;
723 for (
const auto &LC : LibraryCalls) {
755 static const struct {
757 const char *
const Name;
765 for (
const auto &LC : LibraryCalls) {
776 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only() &&
777 Subtarget->hasFPRegs()) {
787 setAllExpand(MVT::f32);
788 if (!Subtarget->hasFP64())
789 setAllExpand(MVT::f64);
792 if (Subtarget->hasFullFP16()) {
801 if (Subtarget->hasBF16()) {
803 setAllExpand(MVT::bf16);
804 if (!Subtarget->hasFullFP16())
811 addAllExtLoads(VT, InnerVT,
Expand);
826 if (Subtarget->hasMVEIntegerOps())
827 addMVEVectorTypes(Subtarget->hasMVEFloatOps());
830 if (Subtarget->hasLOB()) {
834 if (Subtarget->hasNEON()) {
835 addDRTypeForNEON(MVT::v2f32);
836 addDRTypeForNEON(MVT::v8i8);
837 addDRTypeForNEON(MVT::v4i16);
838 addDRTypeForNEON(MVT::v2i32);
839 addDRTypeForNEON(MVT::v1i64);
841 addQRTypeForNEON(MVT::v4f32);
842 addQRTypeForNEON(MVT::v2f64);
843 addQRTypeForNEON(MVT::v16i8);
844 addQRTypeForNEON(MVT::v8i16);
845 addQRTypeForNEON(MVT::v4i32);
846 addQRTypeForNEON(MVT::v2i64);
848 if (Subtarget->hasFullFP16()) {
849 addQRTypeForNEON(MVT::v8f16);
850 addDRTypeForNEON(MVT::v4f16);
853 if (Subtarget->hasBF16()) {
854 addQRTypeForNEON(MVT::v8bf16);
855 addDRTypeForNEON(MVT::v4bf16);
859 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
899 if (Subtarget->hasNEON()) {
1013 for (
MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
1022 for (
auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
1031 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
1039 if (Subtarget->hasMVEIntegerOps()) {
1044 if (Subtarget->hasMVEFloatOps()) {
1048 if (!Subtarget->hasFP64()) {
1095 if (Subtarget->hasFullFP16()) {
1101 if (!Subtarget->hasFP16()) {
1149 if (Subtarget->hasDSP()) {
1171 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6Ops()
1172 || (Subtarget->
isThumb2() && !Subtarget->hasDSP()))
1187 if (Subtarget->hasMVEIntegerOps())
1197 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops())
1208 if (!Subtarget->hasV5TOps() || Subtarget->
isThumb1Only()) {
1217 if (Subtarget->hasPerfMon())
1221 if (!Subtarget->hasV6Ops())
1224 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1225 : Subtarget->hasDivideInARMMode();
1232 if (Subtarget->
isTargetWindows() && !Subtarget->hasDivideInThumbMode()) {
1249 HasStandaloneRem =
false;
1254 const char *
const Name;
1256 } LibraryCalls[] = {
1268 for (
const auto &LC : LibraryCalls) {
1275 const char *
const Name;
1277 } LibraryCalls[] = {
1289 for (
const auto &LC : LibraryCalls) {
1327 InsertFencesForAtomic =
false;
1329 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
1333 if (!Subtarget->isThumb() || !Subtarget->
isMClass())
1338 if (!Subtarget->hasAcquireRelease() ||
1341 InsertFencesForAtomic =
true;
1347 if (Subtarget->hasDataBarrier())
1348 InsertFencesForAtomic =
true;
1368 if (!InsertFencesForAtomic) {
1376 (!Subtarget->
isMClass() && Subtarget->hasV6Ops())) {
1388 }
else if ((Subtarget->
isMClass() && Subtarget->hasV8MBaselineOps()) ||
1389 Subtarget->hasForced32BitAtomics()) {
1403 if (!Subtarget->hasV6Ops()) {
1409 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1441 if (Subtarget->hasFullFP16()) {
1451 if (Subtarget->hasFullFP16())
1466 if (!Subtarget->useSoftFloat() && Subtarget->
hasVFP2Base() &&
1480 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only()) {
1488 if (!Subtarget->hasFP16()) {
1519 if (Subtarget->hasNEON()) {
1526 if (Subtarget->hasFP64()) {
1539 if (Subtarget->hasFullFP16()) {
1558 if (Subtarget->hasNEON()) {
1570 if (Subtarget->hasFullFP16()) {
1602 if (Subtarget->hasMVEIntegerOps())
1605 if (Subtarget->hasV6Ops())
1610 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1617 if (Subtarget->useSoftFloat() || Subtarget->
isThumb1Only() ||
1645 return Subtarget->useSoftFloat();
1658std::pair<const TargetRegisterClass *, uint8_t>
1669 case MVT::f32:
case MVT::f64:
case MVT::v8i8:
case MVT::v4i16:
1670 case MVT::v2i32:
case MVT::v1i64:
case MVT::v2f32:
1671 RRC = &ARM::DPRRegClass;
1679 case MVT::v16i8:
case MVT::v8i16:
case MVT::v4i32:
case MVT::v2i64:
1680 case MVT::v4f32:
case MVT::v2f64:
1681 RRC = &ARM::DPRRegClass;
1685 RRC = &ARM::DPRRegClass;
1689 RRC = &ARM::DPRRegClass;
1693 return std::make_pair(RRC,
Cost);
1697#define MAKE_CASE(V) \
1916 if ((Subtarget->hasMVEIntegerOps() &&
1917 (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
1918 VT == MVT::v16i8)) ||
1919 (Subtarget->hasMVEFloatOps() &&
1920 (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16)))
1934 if (Subtarget->hasNEON()) {
1935 if (VT == MVT::v4i64)
1936 return &ARM::QQPRRegClass;
1937 if (VT == MVT::v8i64)
1938 return &ARM::QQQQPRRegClass;
1940 if (Subtarget->hasMVEIntegerOps()) {
1941 if (VT == MVT::v4i64)
1942 return &ARM::MQQPRRegClass;
1943 if (VT == MVT::v8i64)
1944 return &ARM::MQQQQPRRegClass;
1953 Align &PrefAlign)
const {
1954 if (!isa<MemIntrinsic>(CI))
1972 unsigned NumVals =
N->getNumValues();
1976 for (
unsigned i = 0; i != NumVals; ++i) {
1977 EVT VT =
N->getValueType(i);
1978 if (VT == MVT::Glue || VT == MVT::Other)
1984 if (!
N->isMachineOpcode())
2008 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2009 return Const->getZExtValue() == 16;
2016 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2017 return Const->getZExtValue() == 16;
2024 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2025 return Const->getZExtValue() == 16;
2094 bool isVarArg)
const {
2115 else if (Subtarget->hasFPRegs() && !Subtarget->
isThumb1Only() &&
2136 bool isVarArg)
const {
2137 return CCAssignFnForNode(
CC,
false, isVarArg);
2141 bool isVarArg)
const {
2142 return CCAssignFnForNode(
CC,
true, isVarArg);
2149 bool isVarArg)
const {
2150 switch (getEffectiveCallingConv(
CC, isVarArg)) {
2176 if (Subtarget->hasFullFP16()) {
2189 if (Subtarget->hasFullFP16()) {
2203SDValue ARMTargetLowering::LowerCallResult(
2207 SDValue ThisVal,
bool isCmseNSCall)
const {
2215 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
2220 if (i == 0 && isThisReturn) {
2222 "unexpected return calling convention register assignment");
2233 Chain =
Lo.getValue(1);
2234 InGlue =
Lo.getValue(2);
2238 Chain =
Hi.getValue(1);
2239 InGlue =
Hi.getValue(2);
2251 Chain =
Lo.getValue(1);
2252 InGlue =
Lo.getValue(2);
2255 Chain =
Hi.getValue(1);
2256 InGlue =
Hi.getValue(2);
2300std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
2302 bool IsTailCall,
int SPDiff)
const {
2324 return std::make_pair(DstAddr, DstInfo);
2329 RegsToPassVector &RegsToPass,
2336 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
2337 unsigned id = Subtarget->
isLittle() ? 0 : 1;
2350 std::tie(DstAddr, DstInfo) =
2351 computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
2383 bool isStructRet = (Outs.
empty()) ?
false : Outs[0].
Flags.isSRet();
2384 bool isThisReturn =
false;
2385 bool isCmseNSCall =
false;
2386 bool isSibCall =
false;
2387 bool PreferIndirect =
false;
2388 bool GuardWithBTI =
false;
2398 !Subtarget->noBTIAtReturnTwice())
2403 isCmseNSCall =
true;
2415 if (isa<GlobalAddressSDNode>(Callee)) {
2419 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
2422 PreferIndirect = Subtarget->isThumb() && Subtarget->
hasMinSize() &&
2424 return isa<Instruction>(U) &&
2425 cast<Instruction>(U)->getParent() == BB;
2432 IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs, PreferIndirect);
2446 "site marked musttail");
2449 unsigned NumBytes = CCInfo.getStackSize();
2458 if (isTailCall && !isSibCall) {
2465 NumBytes =
alignTo(NumBytes, StackAlign);
2470 SPDiff = NumReusableBytes - NumBytes;
2474 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (
unsigned)-SPDiff)
2490 RegsToPassVector RegsToPass;
2498 bool AfterFormalArgLoads =
false;
2502 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.
size();
2504 ++i, ++realArgIdx) {
2506 SDValue Arg = OutVals[realArgIdx];
2508 bool isByVal =
Flags.isByVal();
2528 if (isTailCall && VA.
isMemLoc() && !AfterFormalArgLoads) {
2530 AfterFormalArgLoads =
true;
2542 auto ArgVT = Outs[realArgIdx].ArgVT;
2543 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2561 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2562 StackPtr, MemOpChains, isTailCall, SPDiff);
2566 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2567 StackPtr, MemOpChains, isTailCall, SPDiff);
2572 std::tie(DstAddr, DstInfo) =
2573 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2577 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2578 StackPtr, MemOpChains, isTailCall, SPDiff);
2580 if (realArgIdx == 0 &&
Flags.isReturned() && !
Flags.isSwiftSelf() &&
2581 Outs[0].VT == MVT::i32) {
2583 "unexpected calling convention register assignment");
2584 assert(!
Ins.empty() && Ins[0].VT == MVT::i32 &&
2585 "unexpected use of 'returned'");
2586 isThisReturn =
true;
2590 CSInfo.ArgRegPairs.emplace_back(VA.
getLocReg(), i);
2591 RegsToPass.push_back(std::make_pair(VA.
getLocReg(), Arg));
2592 }
else if (isByVal) {
2594 unsigned offset = 0;
2598 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2599 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2601 if (CurByValIdx < ByValArgsCount) {
2603 unsigned RegBegin, RegEnd;
2604 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
2609 for (i = 0, j = RegBegin;
j < RegEnd; i++,
j++) {
2616 RegsToPass.push_back(std::make_pair(j, Load));
2621 offset = RegEnd - RegBegin;
2623 CCInfo.nextInRegsParam();
2626 if (
Flags.getByValSize() > 4*offset) {
2630 std::tie(Dst, DstInfo) =
2631 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2640 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2648 std::tie(DstAddr, DstInfo) =
2649 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2656 if (!MemOpChains.
empty())
2662 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
2663 Chain = DAG.
getCopyToReg(Chain, dl, RegsToPass[i].first,
2664 RegsToPass[i].second, InGlue);
2671 bool isDirect =
false;
2676 GVal =
G->getGlobal();
2677 bool isStub = !
TM.shouldAssumeDSOLocal(GVal) && Subtarget->
isTargetMachO();
2679 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->
isMClass());
2680 bool isLocalARMFunc =
false;
2683 if (Subtarget->genLongCalls()) {
2685 "long-calls codegen is not position independent!");
2689 if (isa<GlobalAddressSDNode>(Callee)) {
2690 if (Subtarget->genExecuteOnly()) {
2709 const char *
Sym = S->getSymbol();
2711 if (Subtarget->genExecuteOnly()) {
2730 }
else if (isa<GlobalAddressSDNode>(Callee)) {
2731 if (!PreferIndirect) {
2736 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !
ARMInterworking);
2738 if (isStub && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2750 "Windows is the only supported COFF target");
2754 else if (!
TM.shouldAssumeDSOLocal(GVal))
2770 const char *
Sym = S->getSymbol();
2771 if (isARMFunc && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2775 ARMPCLabelIndex, 4);
2789 assert(!isARMFunc && !isDirect &&
2790 "Cannot handle call to ARM function or direct call");
2793 "call to non-secure function would "
2794 "require passing arguments on stack",
2801 "call to non-secure function would return value through pointer",
2809 if (Subtarget->isThumb()) {
2812 else if (isCmseNSCall)
2814 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2819 if (!isDirect && !Subtarget->hasV5TOps())
2821 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2834 if (isTailCall && !isSibCall) {
2839 std::vector<SDValue> Ops;
2840 Ops.push_back(Chain);
2841 Ops.push_back(Callee);
2849 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2850 Ops.push_back(DAG.
getRegister(RegsToPass[i].first,
2851 RegsToPass[i].second.getValueType()));
2863 isThisReturn =
false;
2869 assert(Mask &&
"Missing call preserved mask for calling convention");
2873 Ops.push_back(InGlue);
2885 Chain = DAG.
getNode(CallOpc, dl, NodeTys, Ops);
2897 Chain = DAG.
getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
2903 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2904 InVals, isThisReturn,
2905 isThisReturn ? OutVals[0] :
SDValue(), isCmseNSCall);
2912void ARMTargetLowering::HandleByVal(
CCState *State,
unsigned &
Size,
2913 Align Alignment)
const {
2915 Alignment = std::max(Alignment,
Align(4));
2921 unsigned AlignInRegs = Alignment.
value() / 4;
2922 unsigned Waste = (ARM::R4 -
Reg) % AlignInRegs;
2923 for (
unsigned i = 0; i < Waste; ++i)
2929 unsigned Excess = 4 * (ARM::R4 -
Reg);
2936 if (NSAAOffset != 0 &&
Size > Excess) {
2948 unsigned ByValRegBegin =
Reg;
2949 unsigned ByValRegEnd = std::min<unsigned>(Reg +
Size / 4, ARM::R4);
2953 for (
unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2959 Size = std::max<int>(
Size - Excess, 0);
2970 int FI = std::numeric_limits<int>::max();
2978 if (!Flags.isByVal()) {
2984 }
else if (
LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
2985 if (Flags.isByVal())
3000 assert(FI != std::numeric_limits<int>::max());
3011bool ARMTargetLowering::IsEligibleForTailCallOptimization(
3037 if (!isa<GlobalAddressSDNode>(
Callee.getNode()) || isIndirect) {
3039 for (
Register R : {ARM::R0, ARM::R1, ARM::R2, ARM::R3})
3040 AddressRegisters.
insert(R);
3043 AddressRegisters.
insert(ARM::R12);
3046 AddressRegisters.
erase(
AL.getLocReg());
3047 if (AddressRegisters.
empty())
3061 return CalleeCC == CallerCC;
3065 bool isCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
3067 if (isCalleeStructRet || isCallerStructRet)
3081 (!
TT.isOSWindows() ||
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO()))
3088 getEffectiveCallingConv(CalleeCC, isVarArg),
3089 getEffectiveCallingConv(CallerCC, CallerF.
isVarArg()), MF,
C, Ins,
3095 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
3096 if (CalleeCC != CallerCC) {
3097 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
3098 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3111 if (!Outs.
empty()) {
3118 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
3120 ++i, ++realArgIdx) {
3123 SDValue Arg = OutVals[realArgIdx];
3127 if (VA.
needsCustom() && (RegVT == MVT::f64 || RegVT == MVT::v2f64)) {
3134 if (!ArgLocs[++i].isRegLoc())
3136 if (RegVT == MVT::v2f64) {
3137 if (!ArgLocs[++i].isRegLoc())
3139 if (!ArgLocs[++i].isRegLoc())
3164 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
3173 StringRef IntKind =
F.getFnAttribute(
"interrupt").getValueAsString();
3186 if (IntKind ==
"" || IntKind ==
"IRQ" || IntKind ==
"FIQ" ||
3189 else if (IntKind ==
"SWI" || IntKind ==
"UNDEF")
3193 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
3220 bool isLittleEndian = Subtarget->
isLittle();
3232 "secure entry function would return value through pointer",
3238 for (
unsigned i = 0, realRVLocIdx = 0;
3240 ++i, ++realRVLocIdx) {
3244 SDValue Arg = OutVals[realRVLocIdx];
3245 bool ReturnF16 =
false;
3280 auto RetVT = Outs[realRVLocIdx].ArgVT;
3302 DAG.
getVTList(MVT::i32, MVT::i32), Half);
3306 HalfGPRs.
getValue(isLittleEndian ? 0 : 1), Glue);
3312 HalfGPRs.
getValue(isLittleEndian ? 1 : 0), Glue);
3324 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
3326 fmrrd.
getValue(isLittleEndian ? 0 : 1), Glue);
3331 fmrrd.
getValue(isLittleEndian ? 1 : 0), Glue);
3375 return DAG.
getNode(RetNode, dl, MVT::Other, RetOps);
3378bool ARMTargetLowering::isUsedByReturnOnly(
SDNode *
N,
SDValue &Chain)
const {
3379 if (
N->getNumValues() != 1)
3381 if (!
N->hasNUsesOfValue(1, 0))
3389 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3391 TCChain =
Copy->getOperand(0);
3405 SDValue UseChain =
U->getOperand(0);
3413 if (
U->getOperand(
U->getNumOperands() - 1).getValueType() == MVT::Glue)
3421 if (!
Copy->hasOneUse())
3428 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3430 TCChain =
Copy->getOperand(0);
3435 bool HasRet =
false;
3450bool ARMTargetLowering::mayBeEmittedAsTailCall(
const CallInst *CI)
const {
3468 &&
"LowerWRITE_REGISTER called for non-i64 type argument.");
3484 EVT PtrVT =
Op.getValueType();
3494 if (Subtarget->genExecuteOnly()) {
3496 auto T =
const_cast<Type*
>(
CP->getType());
3497 auto C =
const_cast<Constant*
>(
CP->getConstVal());
3508 return LowerGlobalAddress(GA, DAG);
3513 Align CPAlign =
CP->getAlign();
3515 CPAlign = std::max(CPAlign,
Align(4));
3516 if (
CP->isMachineConstantPoolEntry())
3528 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3537 unsigned ARMPCLabelIndex = 0;
3540 const BlockAddress *BA = cast<BlockAddressSDNode>(
Op)->getBlockAddress();
3543 if (!IsPositionIndependent) {
3546 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3557 if (!IsPositionIndependent)
3588ARMTargetLowering::LowerGlobalTLSAddressDarwin(
SDValue Op,
3591 "This function expects a Darwin target");
3596 SDValue DescAddr = LowerGlobalAddressDarwin(
Op, DAG);
3602 MVT::i32,
DL, Chain, DescAddr,
3626 Chain, FuncTLVGet, DAG.
getRegister(ARM::R0, MVT::i32),
3632ARMTargetLowering::LowerGlobalTLSAddressWindows(
SDValue Op,
3649 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
3676 const auto *GA = cast<GlobalAddressSDNode>(
Op);
3693 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3715 Args.push_back(Entry);
3723 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
3724 return CallResult.first;
3746 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3754 PtrVT, dl, Chain,
Offset,
3756 Chain =
Offset.getValue(1);
3762 PtrVT, dl, Chain,
Offset,
3772 PtrVT, dl, Chain,
Offset,
3788 return LowerGlobalTLSAddressDarwin(
Op, DAG);
3791 return LowerGlobalTLSAddressWindows(
Op, DAG);
3800 return LowerToTLSGeneralDynamicModel(GA, DAG);
3803 return LowerToTLSExecModels(GA, DAG, model);
3812 while (!Worklist.
empty()) {
3814 if (isa<ConstantExpr>(U)) {
3819 auto *
I = dyn_cast<Instruction>(U);
3820 if (!
I ||
I->getParent()->getParent() !=
F)
3848 auto *GVar = dyn_cast<GlobalVariable>(GV);
3849 if (!GVar || !GVar->hasInitializer() ||
3850 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3851 !GVar->hasLocalLinkage())
3856 auto *
Init = GVar->getInitializer();
3858 Init->needsDynamicRelocation())
3867 auto *CDAInit = dyn_cast<ConstantDataArray>(
Init);
3870 unsigned RequiredPadding = 4 - (
Size % 4);
3871 bool PaddingPossible =
3872 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3877 unsigned PaddedSize =
Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3901 if (RequiredPadding != 4) {
3906 while (RequiredPadding--)
3918 ++NumConstpoolPromoted;
3923 if (
const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
3924 if (!(GV = GA->getAliaseeObject()))
3926 if (
const auto *V = dyn_cast<GlobalVariable>(GV))
3927 return V->isConstant();
3928 return isa<Function>(GV);
3936 return LowerGlobalAddressWindows(
Op, DAG);
3938 return LowerGlobalAddressELF(
Op, DAG);
3940 return LowerGlobalAddressDarwin(
Op, DAG);
3948 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
3952 if (GV->
isDSOLocal() && !Subtarget->genExecuteOnly())
3965 }
else if (Subtarget->
isROPI() && IsRO) {
3970 }
else if (Subtarget->
isRWPI() && !IsRO) {
3995 if (Subtarget->
useMovt() || Subtarget->genExecuteOnly()) {
4014 "ROPI/RWPI not currently supported for Darwin");
4017 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4040 "Windows on ARM expects to use movw/movt");
4042 "ROPI/RWPI not currently supported for Windows");
4045 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4049 else if (!
TM.shouldAssumeDSOLocal(GV))
4073 DAG.
getVTList(MVT::i32, MVT::Other),
Op.getOperand(0),
4074 Op.getOperand(1), Val);
4091SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
4094 Op.getConstantOperandVal(
Op.getOperand(0).getValueType() == MVT::Other);
4098 case Intrinsic::arm_gnu_eabi_mcount: {
4107 assert(Mask &&
"Missing call preserved mask for calling convention");
4112 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
4116 if (Subtarget->isThumb())
4119 ARM::tBL_PUSHLR, dl, ResultTys,
4120 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
4121 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
4125 {ReturnAddress, Callee, RegisterMask, Chain}),
4134 unsigned IntNo =
Op.getConstantOperandVal(0);
4138 case Intrinsic::thread_pointer: {
4142 case Intrinsic::arm_cls: {
4143 const SDValue &Operand =
Op.getOperand(1);
4144 const EVT VTy =
Op.getValueType();
4155 case Intrinsic::arm_cls64: {
4158 const SDValue &Operand =
Op.getOperand(1);
4159 const EVT VTy =
Op.getValueType();
4182 case Intrinsic::eh_sjlj_lsda: {
4189 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
4199 if (IsPositionIndependent) {
4205 case Intrinsic::arm_neon_vabs:
4208 case Intrinsic::arm_neon_vabds:
4209 if (
Op.getValueType().isInteger())
4211 Op.getOperand(1),
Op.getOperand(2));
4213 case Intrinsic::arm_neon_vabdu:
4215 Op.getOperand(1),
Op.getOperand(2));
4216 case Intrinsic::arm_neon_vmulls:
4217 case Intrinsic::arm_neon_vmullu: {
4218 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
4221 Op.getOperand(1),
Op.getOperand(2));
4223 case Intrinsic::arm_neon_vminnm:
4224 case Intrinsic::arm_neon_vmaxnm: {
4225 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
4228 Op.getOperand(1),
Op.getOperand(2));
4230 case Intrinsic::arm_neon_vminu:
4231 case Intrinsic::arm_neon_vmaxu: {
4232 if (
Op.getValueType().isFloatingPoint())
4234 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
4237 Op.getOperand(1),
Op.getOperand(2));
4239 case Intrinsic::arm_neon_vmins:
4240 case Intrinsic::arm_neon_vmaxs: {
4242 if (!
Op.getValueType().isFloatingPoint()) {
4243 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4246 Op.getOperand(1),
Op.getOperand(2));
4248 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4251 Op.getOperand(1),
Op.getOperand(2));
4253 case Intrinsic::arm_neon_vtbl1:
4255 Op.getOperand(1),
Op.getOperand(2));
4256 case Intrinsic::arm_neon_vtbl2:
4258 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4259 case Intrinsic::arm_mve_pred_i2v:
4260 case Intrinsic::arm_mve_pred_v2i:
4263 case Intrinsic::arm_mve_vreinterpretq:
4266 case Intrinsic::arm_mve_lsll:
4268 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4269 case Intrinsic::arm_mve_asrl:
4271 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4282 if (!Subtarget->hasDataBarrier()) {
4286 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4287 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4298 }
else if (Subtarget->preferISHSTBarriers() &&
4307 DAG.
getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4315 (!Subtarget->
isThumb1Only() && Subtarget->hasV5TEOps())))
4317 return Op.getOperand(0);
4320 unsigned isRead =
~Op.getConstantOperandVal(2) & 1;
4322 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4324 return Op.getOperand(0);
4326 unsigned isData =
Op.getConstantOperandVal(4);
4327 if (Subtarget->isThumb()) {
4329 isRead = ~isRead & 1;
4330 isData = ~isData & 1;
4347 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
4348 return DAG.
getStore(
Op.getOperand(0), dl, FR,
Op.getOperand(1),
4356 const SDLoc &dl)
const {
4362 RC = &ARM::tGPRRegClass;
4364 RC = &ARM::GPRRegClass;
4378 MVT::i32, dl, Root, FIN,
4399 const Value *OrigArg,
4400 unsigned InRegsParamRecordIdx,
4401 int ArgOffset,
unsigned ArgSize)
const {
4416 unsigned RBegin, REnd;
4426 ArgOffset = -4 * (ARM::R4 - RBegin);
4436 for (
unsigned Reg = RBegin, i = 0;
Reg < REnd; ++
Reg, ++i) {
4445 if (!MemOps.
empty())
4454 unsigned TotalArgRegsSaveSize,
4455 bool ForceMutable)
const {
4466 CCInfo.
getStackSize(), std::max(4U, TotalArgRegsSaveSize));
4470bool ARMTargetLowering::splitValueIntoRegisterParts(
4472 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
4474 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4486SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4488 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
4489 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4502SDValue ARMTargetLowering::LowerFormalArguments(
4518 unsigned CurArgIdx = 0;
4530 unsigned ArgRegBegin = ARM::R4;
4537 if (!
Flags.isByVal())
4541 unsigned RBegin, REnd;
4543 ArgRegBegin = std::min(ArgRegBegin, RBegin);
4549 int lastInsIndex = -1;
4553 ArgRegBegin = std::min(ArgRegBegin, (
unsigned)
GPRArgRegs[RegIdx]);
4556 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4560 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4562 if (Ins[VA.
getValNo()].isOrigArg()) {
4563 std::advance(CurOrigArg,
4564 Ins[VA.
getValNo()].getOrigArgIndex() - CurArgIdx);
4576 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4583 MVT::f64, dl, Chain, FIN,
4586 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4594 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4598 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4599 RC = &ARM::HPRRegClass;
4600 else if (RegVT == MVT::f32)
4601 RC = &ARM::SPRRegClass;
4602 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4603 RegVT == MVT::v4bf16)
4604 RC = &ARM::DPRRegClass;
4605 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4606 RegVT == MVT::v8bf16)
4607 RC = &ARM::QPRRegClass;
4608 else if (RegVT == MVT::i32)
4610 : &ARM::GPRRegClass;
4656 assert(VA.
getValVT() != MVT::i64 &&
"i64 should already be lowered");
4662 if (index != lastInsIndex)
4670 if (
Flags.isByVal()) {
4671 assert(Ins[index].isOrigArg() &&
4672 "Byval arguments cannot be implicit");
4676 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4691 lastInsIndex = index;
4698 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.
getStackSize(),
4699 TotalArgRegsSaveSize);
4703 "secure entry function must not be variadic", dl.
getDebugLoc());
4714 StackArgSize =
alignTo(StackArgSize,
DL.getStackAlignment());
4723 "secure entry function requires arguments on stack", dl.
getDebugLoc());
4733 return CFP->getValueAPF().isPosZero();
4737 SDValue WrapperOp =
Op.getOperand(1).getOperand(0);
4739 if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
4740 return CFP->getValueAPF().isPosZero();
4743 Op->getValueType(0) == MVT::f64) {
4758 const SDLoc &dl)
const {
4760 unsigned C = RHSC->getZExtValue();
4826 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(RHS) &&
4828 unsigned Mask =
LHS.getConstantOperandVal(1);
4829 auto *RHSC = cast<ConstantSDNode>(
RHS.getNode());
4830 uint64_t RHSV = RHSC->getZExtValue();
4831 if (
isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4833 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4848 isa<ConstantSDNode>(RHS) &&
RHS->getAsZExtVal() == 0x80000000U &&
4850 LHS.getConstantOperandVal(1) < 31) {
4851 unsigned ShiftAmt =
LHS.getConstantOperandVal(1) + 1;
4891 return DAG.
getNode(CompareType, dl, MVT::Glue, LHS, RHS);
4897 bool Signaling)
const {
4898 assert(Subtarget->hasFP64() ||
RHS.getValueType() != MVT::f64);
4902 dl, MVT::Glue, LHS, RHS);
4905 dl, MVT::Glue, LHS);
4913 unsigned Opc =
Cmp.getOpcode();
4916 return DAG.
getNode(Opc,
DL, MVT::Glue,
Cmp.getOperand(0),
Cmp.getOperand(1));
4920 Opc =
Cmp.getOpcode();
4934std::pair<SDValue, SDValue>
4937 assert(
Op.getValueType() == MVT::i32 &&
"Unsupported value type");
4949 switch (
Op.getOpcode()) {
4962 DAG.
getVTList(
Op.getValueType(), MVT::i32), LHS, RHS)
5001 return std::make_pair(
Value, OverflowCmp);
5012 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Op, DAG, ARMcc);
5018 EVT VT =
Op.getValueType();
5021 ARMcc, CCR, OverflowCmp);
5061 EVT VT =
Op.getValueType();
5065 switch (
Op.getOpcode()) {
5090 EVT VT =
Op.getValueType();
5091 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->
isThumb1Only())
5101 switch (
Op->getOpcode()) {
5117 switch (
Op->getOpcode()) {
5136 DAG.
getNode(NewOpcode, dl, MVT::i32,
5147 unsigned Opc =
Cond.getOpcode();
5149 if (
Cond.getResNo() == 1 &&
5157 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5159 EVT VT =
Op.getValueType();
5161 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
5172 dyn_cast<ConstantSDNode>(
Cond.getOperand(0));
5174 dyn_cast<ConstantSDNode>(
Cond.getOperand(1));
5176 if (CMOVTrue && CMOVFalse) {
5182 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5184 False = SelectFalse;
5185 }
else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5191 EVT VT =
Op.getValueType();
5196 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG);
5212 bool &swpCmpOps,
bool &swpVselOps) {
5240 swpCmpOps = !swpCmpOps;
5241 swpVselOps = !swpVselOps;
5264 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5266 DAG.
getVTList(MVT::i32, MVT::i32), FalseVal);
5268 DAG.
getVTList(MVT::i32, MVT::i32), TrueVal);
5278 ARMcc, CCR, duplicateCmp(Cmp, DAG));
5305 ((K ==
LHS && K == TrueVal) || (K ==
RHS && K == FalseVal))) ||
5307 ((K ==
RHS && K == TrueVal) || (K ==
LHS && K == FalseVal)));
5328 EVT VT =
Op.getValueType();
5335 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
5350 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5358 if (!isa<ConstantSDNode>(K1) || !isa<ConstantSDNode>(K2))
5361 int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue();
5362 int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue();
5363 int64_t PosVal = std::max(Val1, Val2);
5364 int64_t NegVal = std::min(Val1, Val2);
5410 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
5411 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5416 if (*K != KTmp || V != VTmp)
5427bool ARMTargetLowering::isUnsupportedFloatingType(
EVT VT)
const {
5431 return !Subtarget->hasFP64();
5433 return !Subtarget->hasFullFP16();
5438 EVT VT =
Op.getValueType();
5442 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->
isThumb2())
5454 if (VT == MVT::i32 &&
5474 if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
5475 LHS.getValueType() == MVT::i32 &&
RHS.getValueType() == MVT::i32) {
5478 unsigned Opcode = 0;
5480 if (TVal == ~FVal) {
5482 }
else if (TVal == ~FVal + 1) {
5484 }
else if (TVal + 1 == FVal) {
5486 }
else if (TVal == FVal + 1) {
5518 return DAG.
getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
5522 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5524 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5528 if (!
RHS.getNode()) {
5534 if (
LHS.getValueType() == MVT::i32) {
5546 TrueVal.getValueType() == MVT::f32 ||
5547 TrueVal.getValueType() == MVT::f64)) {
5562 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5574 (
TrueVal.getValueType() == MVT::f16 ||
5575 TrueVal.getValueType() == MVT::f32 ||
5576 TrueVal.getValueType() == MVT::f64)) {
5577 bool swpCmpOps =
false;
5578 bool swpVselOps =
false;
5593 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5597 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
5598 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
5608 if (!
N->hasOneUse())
5611 if (!
N->getNumValues())
5613 EVT VT =
Op.getValueType();
5614 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5631 return DAG.
getLoad(MVT::i32,
SDLoc(
Op), Ld->getChain(), Ld->getBasePtr(),
5632 Ld->getPointerInfo(), Ld->getAlign(),
5633 Ld->getMemOperand()->getFlags());
5651 DAG.
getLoad(MVT::i32, dl, Ld->getChain(),
Ptr, Ld->getPointerInfo(),
5652 Ld->getAlign(), Ld->getMemOperand()->getFlags());
5654 EVT PtrType =
Ptr.getValueType();
5657 RetVal2 = DAG.
getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5658 Ld->getPointerInfo().getWithOffset(4),
5660 Ld->getMemOperand()->getFlags());
5678 bool LHSSeenZero =
false;
5680 bool RHSSeenZero =
false;
5682 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5693 if (
LHS.getValueType() == MVT::f32) {
5701 Chain, Dest, ARMcc, CCR, Cmp);
5713 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5728 unsigned Opc =
Cond.getOpcode();
5731 if (
Cond.getResNo() == 1 &&
5741 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5765 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5767 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5771 if (!
RHS.getNode()) {
5779 unsigned Opc =
LHS.getOpcode();
5793 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
LHS.getValue(0), DAG, ARMcc);
5808 if (
LHS.getValueType() == MVT::i32) {
5813 Chain, Dest, ARMcc, CCR, Cmp);
5819 if (
SDValue Result = OptimizeVFPBrcond(
Op, DAG))
5830 SDValue Ops[] = { Chain, Dest, ARMcc, CCR,
Cmp };
5852 if (Subtarget->
isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5858 Addr,
Op.getOperand(2), JTI);
5864 Chain =
Addr.getValue(1);
5871 Chain =
Addr.getValue(1);
5877 EVT VT =
Op.getValueType();
5880 if (
Op.getValueType().getVectorElementType() == MVT::i32) {
5881 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
5889 const EVT OpTy =
Op.getOperand(0).getValueType();
5890 if (OpTy == MVT::v4f32)
5892 else if (OpTy == MVT::v4f16 && HasFullFP16)
5894 else if (OpTy == MVT::v8f16 && HasFullFP16)
5899 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5902 Op = DAG.
getNode(
Op.getOpcode(), dl, NewTy,
Op.getOperand(0));
5907 EVT VT =
Op.getValueType();
5911 bool IsStrict =
Op->isStrictFPOpcode();
5912 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
5914 if (isUnsupportedFloatingType(SrcVal.
getValueType())) {
5924 MakeLibCallOptions CallOptions;
5927 std::tie(Result, Chain) =
makeLibCall(DAG, LC,
Op.getValueType(), SrcVal,
5928 CallOptions, Loc, Chain);
5938 Loc,
Op.getValueType(), SrcVal);
5947 EVT VT =
Op.getValueType();
5948 EVT ToVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
5949 EVT FromVT =
Op.getOperand(0).getValueType();
5951 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5953 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5954 Subtarget->hasFP64())
5956 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5957 Subtarget->hasFullFP16())
5959 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5960 Subtarget->hasMVEFloatOps())
5962 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5963 Subtarget->hasMVEFloatOps())
5966 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5983 EVT VT =
Op.getValueType();
5986 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
5992 assert((
Op.getOperand(0).getValueType() == MVT::v4i16 ||
5993 Op.getOperand(0).getValueType() == MVT::v8i16) &&
5994 "Invalid type for custom lowering!");
5999 if (VT == MVT::v4f32)
6000 DestVecType = MVT::v4i32;
6001 else if (VT == MVT::v4f16 && HasFullFP16)
6002 DestVecType = MVT::v4i16;
6003 else if (VT == MVT::v8f16 && HasFullFP16)
6004 DestVecType = MVT::v8i16;
6010 switch (
Op.getOpcode()) {
6022 Op = DAG.
getNode(CastOpc, dl, DestVecType,
Op.getOperand(0));
6027 EVT VT =
Op.getValueType();
6030 if (isUnsupportedFloatingType(VT)) {
6038 MakeLibCallOptions CallOptions;
6040 CallOptions,
SDLoc(
Op)).first;
6051 EVT VT =
Op.getValueType();
6055 bool UseNEON = !InGPR && Subtarget->hasNEON();
6062 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
6069 if (SrcVT == MVT::f32) {
6075 }
else if (VT == MVT::f32)
6091 if (VT == MVT::f32) {
6103 if (SrcVT == MVT::f64)
6112 if (VT == MVT::f32) {
6136 EVT VT =
Op.getValueType();
6138 unsigned Depth =
Op.getConstantOperandVal(0);
6140 SDValue FrameAddr = LowerFRAMEADDR(
Op, DAG);
6159 EVT VT =
Op.getValueType();
6161 unsigned Depth =
Op.getConstantOperandVal(0);
6175 .
Case(
"sp", ARM::SP)
6190 assert(
N->getValueType(0) == MVT::i64
6191 &&
"ExpandREAD_REGISTER called for non-i64 type result.");
6194 DAG.
getVTList(MVT::i32, MVT::i32, MVT::Other),
6200 Results.push_back(Read.getOperand(0));
6234 const APInt &APIntIndex =
Index->getAPIntValue();
6236 NewIndex *= APIntIndex;
6266 EVT SrcVT =
Op.getValueType();
6267 EVT DstVT =
N->getValueType(0);
6269 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6270 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6274 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6275 (SrcVT == MVT::f16 || SrcVT == MVT::bf16))
6280 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6284 if (SrcVT == MVT::i64 && TLI.
isTypeLegal(DstVT)) {
6296 if (DstVT == MVT::i64 && TLI.
isTypeLegal(SrcVT)) {
6333 EVT VT =
Op.getValueType();
6352 SDValue LoBigShift = DAG.
getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6360 ? DAG.
getNode(Opc, dl, VT, ShOpHi,
6377 EVT VT =
Op.getValueType();
6420 DAG.
getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6472 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6500 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6530 EVT VT =
N->getValueType(0);
6531 if (VT.
isVector() && ST->hasNEON()) {
6540 if (ElemTy == MVT::i8) {
6548 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6563 if (ElemTy == MVT::i64) {
6576 if (!ST->hasV6T2Ops())
6585 EVT VT =
N->getValueType(0);
6588 assert(ST->hasNEON() &&
"Custom ctpop lowering requires NEON.");
6589 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6590 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6591 "Unexpected type for custom ctpop lowering");
6599 unsigned EltSize = 8;
6622 Op =
Op.getOperand(0);
6624 APInt SplatBits, SplatUndef;
6625 unsigned SplatBitSize;
6628 !BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6630 SplatBitSize > ElementBits)
6641 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6645 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6656 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6661 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6662 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6671 EVT VT =
N->getValueType(0);
6693 "unexpected vector shift opcode");
6695 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
6696 unsigned VShiftOpc =
6698 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
6704 EVT ShiftVT =
N->getOperand(1).getValueType();
6707 unsigned VShiftOpc =
6709 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0), NegatedCount);
6714 EVT VT =
N->getValueType(0);
6723 "Unknown shift to lower!");
6725 unsigned ShOpc =
N->getOpcode();
6726 if (ST->hasMVEIntegerOps()) {
6756 DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6770 if (ST->isThumb1Only())
6775 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6791 bool Invert =
false;
6798 EVT VT =
Op.getValueType();
6806 assert(ST->hasMVEIntegerOps() &&
6807 "No hardware support for integer vector comparison!");
6809 if (
Op.getValueType().getVectorElementType() != MVT::i1)
6834 Merged = DAG.
getNOT(dl, Merged, CmpVT);
6844 switch (SetCCOpcode) {
6848 if (ST->hasMVEFloatOps()) {
6851 Invert =
true; [[fallthrough]];
6856 case ISD::SETLT: Swap =
true; [[fallthrough]];
6860 case ISD::SETLE: Swap =
true; [[fallthrough]];
6876 Result = DAG.
getNOT(dl, Result, VT);
6879 case ISD::SETUO: Invert =
true; [[fallthrough]];
6888 Result = DAG.
getNOT(dl, Result, VT);
6894 switch (SetCCOpcode) {
6897 if (ST->hasMVEIntegerOps()) {
6900 Invert =
true; [[fallthrough]];
6903 case ISD::SETLT: Swap =
true; [[fallthrough]];
6905 case ISD::SETLE: Swap =
true; [[fallthrough]];
6914 if (ST->hasNEON() && Opc ==
ARMCC::EQ) {
6930 Result = DAG.
getNOT(dl, Result, VT);
6964 Result = DAG.
getNOT(dl, Result, VT);
6976 assert(
LHS.getSimpleValueType().isInteger() &&
"SETCCCARRY is integer only.");
7006 unsigned OpCmode, Imm;
7017 switch (SplatBitSize) {
7022 assert((SplatBits & ~0xff) == 0 &&
"one byte splat value is too big");
7025 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
7030 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
7031 if ((SplatBits & ~0xff) == 0) {
7037 if ((SplatBits & ~0xff00) == 0) {
7040 Imm = SplatBits >> 8;
7050 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
7051 if ((SplatBits & ~0xff) == 0) {
7057 if ((SplatBits & ~0xff00) == 0) {
7060 Imm = SplatBits >> 8;
7063 if ((SplatBits & ~0xff0000) == 0) {
7066 Imm = SplatBits >> 16;
7069 if ((SplatBits & ~0xff000000) == 0) {
7072 Imm = SplatBits >> 24;
7079 if ((SplatBits & ~0xffff) == 0 &&
7080 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
7083 Imm = SplatBits >> 8;
7091 if ((SplatBits & ~0xffffff) == 0 &&
7092 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
7095 Imm = SplatBits >> 16;
7111 unsigned ImmMask = 1;
7113 for (
int ByteNum = 0; ByteNum < 8; ++ByteNum) {
7114 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
7116 }
else if ((SplatBits & BitMask) != 0) {
7126 unsigned Mask = (1 << BytesPerElem) - 1;
7127 unsigned NumElems = 8 / BytesPerElem;
7128 unsigned NewImm = 0;
7129 for (
unsigned ElemNum = 0; ElemNum < NumElems; ++ElemNum) {
7130 unsigned Elem = ((Imm >> ElemNum * BytesPerElem) & Mask);
7131 NewImm |= Elem << (NumElems - ElemNum - 1) * BytesPerElem;
7138 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
7152 EVT VT =
Op.getValueType();
7153 bool IsDouble = (VT == MVT::f64);
7159 if (
ST->genExecuteOnly()) {
7161 assert((!
ST->isThumb1Only() ||
ST->hasV8MBaselineOps()) &&
7162 "Unexpected architecture");
7185 if (!
ST->hasVFP3Base())
7190 if (IsDouble && !Subtarget->hasFP64())
7197 if (IsDouble || !
ST->useNEONForSinglePrecisionFP()) {
7215 if (!
ST->hasNEON() || (!IsDouble && !
ST->useNEONForSinglePrecisionFP()))
7224 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7278 unsigned ExpectedElt = Imm;
7279 for (
unsigned i = 1; i < NumElts; ++i) {
7283 if (ExpectedElt == NumElts)
7286 if (M[i] < 0)
continue;
7287 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7295 bool &ReverseVEXT,
unsigned &Imm) {
7297 ReverseVEXT =
false;
7308 unsigned ExpectedElt = Imm;
7309 for (
unsigned i = 1; i < NumElts; ++i) {
7313 if (ExpectedElt == NumElts * 2) {
7318 if (M[i] < 0)
continue;
7319 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7334 return VT == MVT::v8i8 && M.size() == 8;
7339 if (Mask.size() == Elements * 2)
7340 return Index / Elements;
7341 return Mask[
Index] == 0 ? 0 : 1;
7371 if (M.size() != NumElts && M.size() != NumElts*2)
7379 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7381 for (
unsigned j = 0; j < NumElts; j += 2) {
7382 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7383 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + NumElts + WhichResult))
7388 if (M.size() == NumElts*2)
7403 if (M.size() != NumElts && M.size() != NumElts*2)
7406 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7408 for (
unsigned j = 0; j < NumElts; j += 2) {
7409 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7410 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + WhichResult))
7415 if (M.size() == NumElts*2)
7435 if (M.size() != NumElts && M.size() != NumElts*2)
7438 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7440 for (
unsigned j = 0; j < NumElts; ++j) {
7441 if (M[i+j] >= 0 && (
unsigned) M[i+j] != 2 * j + WhichResult)
7446 if (M.size() == NumElts*2)
7465 if (M.size() != NumElts && M.size() != NumElts*2)
7468 unsigned Half = NumElts / 2;
7469 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7471 for (
unsigned j = 0; j < NumElts; j += Half) {
7472 unsigned Idx = WhichResult;
7473 for (
unsigned k = 0; k < Half; ++k) {
7474 int MIdx = M[i + j + k];
7475 if (MIdx >= 0 && (
unsigned) MIdx !=
Idx)
7482 if (M.size() == NumElts*2)
7506 if (M.size() != NumElts && M.size() != NumElts*2)
7509 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7511 unsigned Idx = WhichResult * NumElts / 2;
7512 for (
unsigned j = 0; j < NumElts; j += 2) {
7513 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7514 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx + NumElts))
7520 if (M.size() == NumElts*2)
7539 if (M.size() != NumElts && M.size() != NumElts*2)
7542 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7544 unsigned Idx = WhichResult * NumElts / 2;
7545 for (
unsigned j = 0; j < NumElts; j += 2) {
7546 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7547 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx))
7553 if (M.size() == NumElts*2)
7566 unsigned &WhichResult,
7569 if (
isVTRNMask(ShuffleMask, VT, WhichResult))
7571 if (
isVUZPMask(ShuffleMask, VT, WhichResult))
7573 if (
isVZIPMask(ShuffleMask, VT, WhichResult))
7591 if (NumElts != M.size())
7595 for (
unsigned i = 0; i != NumElts; ++i)
7596 if (M[i] >= 0 && M[i] != (
int) (NumElts - 1 - i))
7605 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7613 int Ofs = Top ? 1 : 0;
7614 int Upper = SingleSource ? 0 : NumElts;
7615 for (
int i = 0, e = NumElts / 2; i != e; ++i) {
7616 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7618 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs +
Upper)
7627 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7636 unsigned Offset = Top ? 0 : 1;
7637 unsigned N = SingleSource ? 0 : NumElts;
7638 for (
unsigned i = 0; i < NumElts; i += 2) {
7639 if (M[i] >= 0 && M[i] != (
int)i)
7641 if (M[i + 1] >= 0 && M[i + 1] != (
int)(
N + i +
Offset))
7650 if (NumElts != M.size())
7658 unsigned Off0 = rev ? NumElts / 2 : 0;
7659 unsigned Off1 = rev ? 0 : NumElts / 2;
7660 for (
unsigned i = 0; i < NumElts; i += 2) {
7661 if (M[i] >= 0 && M[i] != (
int)(Off0 + i / 2))
7663 if (M[i + 1] >= 0 && M[i + 1] != (
int)(Off1 + i / 2))
7679 if (!ST->hasMVEFloatOps())
7684 if (VT != MVT::v8f16)
7705 for (
unsigned i = 1; i < 4; i++) {
7732 if (!ST->hasMVEFloatOps())
7737 if (VT != MVT::v4f32)
7753 for (
unsigned i = 1; i < 4; i++) {
7774 if (!isa<ConstantSDNode>(
N))
7776 Val =
N->getAsZExtVal();
7778 if (ST->isThumb1Only()) {
7779 if (Val <= 255 || ~Val <= 255)
7791 EVT VT =
Op.getValueType();
7793 assert(ST->hasMVEIntegerOps() &&
"LowerBUILD_VECTOR_i1 called without MVE!");
7797 unsigned BitsPerBool;
7801 }
else if (NumElts == 4) {
7804 }
else if (NumElts == 8) {
7807 }
else if (NumElts == 16) {
7816 if (!isa<ConstantSDNode>(FirstOp) &&
7818 return U.get().isUndef() || U.get() == FirstOp;
7826 unsigned Bits32 = 0;
7827 for (
unsigned i = 0; i < NumElts; ++i) {
7829 if (!isa<ConstantSDNode>(V) && !V.isUndef())
7831 bool BitSet = V.isUndef() ?
false : V->getAsZExtVal();
7833 Bits32 |= BoolMask << (i * BitsPerBool);
7839 for (
unsigned i = 0; i < NumElts; ++i) {
7841 if (isa<ConstantSDNode>(V) || V.isUndef())
7852 if (!ST->hasMVEIntegerOps())
7856 EVT VT =
Op.getValueType();
7866 if (
N != 1 &&
N != 2 &&
N != 4 &&
N != 8)
7870 for (
unsigned I = 2;
I < NumElts;
I++) {
7886 switch (
N->getOpcode()) {
7895 return N->getOperand(1).getNode() ==
Op;
7897 switch (
N->getConstantOperandVal(0)) {
7898 case Intrinsic::arm_mve_add_predicated:
7899 case Intrinsic::arm_mve_mul_predicated:
7900 case Intrinsic::arm_mve_qadd_predicated:
7901 case Intrinsic::arm_mve_vhadd:
7902 case Intrinsic::arm_mve_hadd_predicated:
7903 case Intrinsic::arm_mve_vqdmulh:
7904 case Intrinsic::arm_mve_qdmulh_predicated:
7905 case Intrinsic::arm_mve_vqrdmulh:
7906 case Intrinsic::arm_mve_qrdmulh_predicated:
7907 case Intrinsic::arm_mve_vqdmull:
7908 case Intrinsic::arm_mve_vqdmull_predicated:
7910 case Intrinsic::arm_mve_sub_predicated:
7911 case Intrinsic::arm_mve_qsub_predicated:
7912 case Intrinsic::arm_mve_vhsub:
7913 case Intrinsic::arm_mve_hsub_predicated:
7914 return N->getOperand(2).getNode() ==
Op;
7929 EVT VT =
Op.getValueType();
7937 APInt SplatBits, SplatUndef;
7938 unsigned SplatBitSize;
7940 if (BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7947 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7949 [BVN](
const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
7950 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7951 : SplatBitSize == 16 ? MVT::v8i16
7958 if ((
ST->hasNEON() && SplatBitSize <= 64) ||
7959 (
ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7964 SplatBitSize, DAG, dl, VmovVT, VT,
VMOVModImm);
7972 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7974 NegatedImm, SplatUndef.
getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
7982 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7992 if (
ST->hasMVEIntegerOps() &&
7993 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
7994 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7995 : SplatBitSize == 16 ? MVT::v8i16
8012 bool isOnlyLowElement =
true;
8013 bool usesOnlyOneValue =
true;
8014 bool hasDominantValue =
false;
8021 for (
unsigned i = 0; i < NumElts; ++i) {
8026 isOnlyLowElement =
false;
8027 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
8030 ValueCounts.
insert(std::make_pair(V, 0));
8031 unsigned &Count = ValueCounts[
V];
8034 if (++Count > (NumElts / 2)) {
8035 hasDominantValue =
true;
8039 if (ValueCounts.
size() != 1)
8040 usesOnlyOneValue =
false;
8041 if (!
Value.getNode() && !ValueCounts.
empty())
8044 if (ValueCounts.
empty())
8056 if (hasDominantValue && EltSize <= 32) {
8067 (constIndex = dyn_cast<ConstantSDNode>(
Value->getOperand(1)))) {
8072 if (VT !=
Value->getOperand(0).getValueType()) {
8085 if (!usesOnlyOneValue) {
8088 for (
unsigned I = 0;
I < NumElts; ++
I) {
8103 assert(FVT == MVT::f32 || FVT == MVT::f16);
8104 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
8105 for (
unsigned i = 0; i < NumElts; ++i)
8110 Val = LowerBUILD_VECTOR(Val, DAG, ST);
8114 if (usesOnlyOneValue) {
8141 if (
ST->hasNEON() && VT.
is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
8161 if (EltSize >= 32) {
8167 for (
unsigned i = 0; i < NumElts; ++i)
8181 for (
unsigned i = 0 ; i < NumElts; ++i) {
8200 EVT VT =
Op.getValueType();
8203 struct ShuffleSourceInfo {
8205 unsigned MinElt = std::numeric_limits<unsigned>::max();
8206 unsigned MaxElt = 0;
8216 int WindowScale = 1;
8218 ShuffleSourceInfo(
SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
8220 bool operator ==(
SDValue OtherVec) {
return Vec == OtherVec; }
8226 for (
unsigned i = 0; i < NumElts; ++i) {
8234 }
else if (!isa<ConstantSDNode>(
V.getOperand(1))) {
8241 SDValue SourceVec =
V.getOperand(0);
8243 if (Source == Sources.
end())
8247 unsigned EltNo =
V.getConstantOperandVal(1);
8254 if (Sources.
size() > 2)
8260 for (
auto &Source : Sources) {
8261 EVT SrcEltTy =
Source.Vec.getValueType().getVectorElementType();
8262 if (SrcEltTy.
bitsLT(SmallestEltTy))
8263 SmallestEltTy = SrcEltTy;
8265 unsigned ResMultiplier =
8273 for (
auto &Src : Sources) {
8274 EVT SrcVT = Src.ShuffleVec.getValueType();
8278 if (SrcVTSize == VTSize)
8287 if (SrcVTSize < VTSize) {
8288 if (2 * SrcVTSize != VTSize)
8294 DAG.
getUNDEF(Src.ShuffleVec.getValueType()));
8298 if (SrcVTSize != 2 * VTSize)
8301 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8306 if (Src.MinElt >= NumSrcElts) {
8311 Src.WindowBase = -NumSrcElts;
8312 }
else if (Src.MaxElt < NumSrcElts) {
8329 Src.WindowBase = -Src.MinElt;
8336 for (
auto &Src : Sources) {
8337 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8338 if (SrcEltTy == SmallestEltTy)
8343 Src.WindowBase *= Src.WindowScale;
8349 assert(Src.ShuffleVec.getValueType() == ShuffleVT););
8356 if (
Entry.isUndef())
8360 int EltNo = cast<ConstantSDNode>(
Entry.getOperand(1))->getSExtValue();
8365 EVT OrigEltTy =
Entry.getOperand(0).getValueType().getVectorElementType();
8368 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8372 int *LaneMask = &
Mask[i * ResMultiplier];
8374 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8375 ExtractBase += NumElts * (Src - Sources.begin());
8376 for (
int j = 0;
j < LanesDefined; ++
j)
8377 LaneMask[j] = ExtractBase + j;
8383 assert(Sources.size() <= 2 &&
"Too many sources!");
8386 for (
unsigned i = 0; i < Sources.size(); ++i)
8415 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8435 unsigned PFIndexes[4];
8436 for (
unsigned i = 0; i != 4; ++i) {
8440 PFIndexes[i] = M[i];
8444 unsigned PFTableIndex =
8445 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8447 unsigned Cost = (PFEntry >> 30);
8453 bool ReverseVEXT, isV_UNDEF;
8454 unsigned Imm, WhichResult;
8457 if (EltSize >= 32 ||
8464 else if (Subtarget->hasNEON() &&
8469 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8472 else if (Subtarget->hasMVEIntegerOps() &&
8476 else if (Subtarget->hasMVEIntegerOps() &&
8490 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8491 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8492 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8495 if (LHSID == (1*9+2)*9+3)
return LHS;
8496 assert(LHSID == ((4*9+5)*9+6)*9+7 &&
"Illegal OP_COPY!");
8553 for (
int I : ShuffleMask)
8556 if (V2.getNode()->isUndef())
8566 EVT VT =
Op.getValueType();
8568 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8569 "Expect an v8i16/v16i8 type");
8575 std::vector<int> NewMask;
8579 NewMask.push_back(i);
8619 if (VT != MVT::v16i1)
8635 EVT VT =
Op.getValueType();
8639 assert(ST->hasMVEIntegerOps() &&
8640 "No support for vector shuffle of boolean predicates");
8666 "Expected identical vector type in expanded i1 shuffle!");
8670 PredAsVector2, ShuffleMask);
8675 if (VT == MVT::v2i1) {
8692 EVT VT =
Op.getValueType();
8696 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8697 "Unexpected vector type");
8699 int QuarterSize = NumElts / 4;
8708 for (
int i = 0; i <
Length; i++) {
8709 if (ShuffleMask[Start + i] >= 0) {
8710 if (ShuffleMask[Start + i] %
Length != i)
8712 MovIdx = ShuffleMask[Start + i] /
Length;
8720 for (
int i = 1; i <
Length; i++) {
8721 if (ShuffleMask[Start + i] >= 0 &&
8722 (ShuffleMask[Start + i] /
Length != MovIdx ||
8723 ShuffleMask[Start + i] %
Length != i))
8729 for (
int Part = 0; Part < 4; ++Part) {
8731 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8735 Input =
Op->getOperand(1);
8745 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8750 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8752 for (
int Part = 0; Part < 4; ++Part)
8753 for (
int i = 0; i < QuarterSize; i++)
8755 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8757 VT, dl,
Op->getOperand(0),
Op->getOperand(1), NewShuffleMask);
8760 for (
int Part = 0; Part < 4; ++Part)
8776 EVT VT =
Op.getValueType();
8788 for (
int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8792 if (Mask[i] != i + BaseOffset) {
8793 if (OffElement == -1)
8799 return NonUndef > 2 && OffElement != -1;
8803 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8805 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8816 ShuffleMask[OffElement] < (
int)NumElts ? V1 : V2,
8827 EVT VT =
Op.getValueType();
8831 if (ST->hasMVEIntegerOps() && EltSize == 1)
8842 if (EltSize <= 32) {
8846 if (Lane == -1) Lane = 0;
8857 bool IsScalarToVector =
true;
8860 IsScalarToVector =
false;
8863 if (IsScalarToVector)
8870 bool ReverseVEXT =
false;
8872 if (ST->hasNEON() &&
isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
8896 unsigned WhichResult = 0;
8897 bool isV_UNDEF =
false;
8898 if (ST->hasNEON()) {
8900 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8907 if (ST->hasMVEIntegerOps()) {
8942 }) &&
"Unexpected shuffle index into UNDEF operand!");
8945 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
8948 assert((WhichResult == 0) &&
8949 "In-place shuffle of concat can only have one result!");
8958 if (ST->hasMVEIntegerOps() && EltSize <= 32) {
8962 for (
bool Top : {
false,
true}) {
8963 for (
bool SingleSource : {
false,
true}) {
8964 if (
isTruncMask(ShuffleMask, VT, Top, SingleSource)) {
8969 SingleSource ? V1 : V2);
8985 unsigned PFIndexes[4];
8986 for (
unsigned i = 0; i != 4; ++i) {
8987 if (ShuffleMask[i] < 0)
8990 PFIndexes[i] = ShuffleMask[i];
8994 unsigned PFTableIndex =
8995 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8997 unsigned Cost = (PFEntry >> 30);
9003 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
9004 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
9014 if (EltSize >= 32) {
9022 for (
unsigned i = 0; i < NumElts; ++i) {
9023 if (ShuffleMask[i] < 0)
9027 ShuffleMask[i] < (
int)NumElts ? V1 : V2,
9035 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
9039 if (ST->hasNEON() && VT == MVT::v8i8)
9043 if (ST->hasMVEIntegerOps())
9052 EVT VecVT =
Op.getOperand(0).getValueType();
9055 assert(ST->hasMVEIntegerOps() &&
9056 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9060 unsigned Lane =
Op.getConstantOperandVal(2);
9061 unsigned LaneWidth =
9063 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
9075 if (!isa<ConstantSDNode>(Lane))
9081 if (Subtarget->hasMVEIntegerOps() &&
9082 Op.getValueType().getScalarSizeInBits() == 1)
9106 IVecIn, IElt, Lane);
9115 EVT VecVT =
Op.getOperand(0).getValueType();
9118 assert(ST->hasMVEIntegerOps() &&
9119 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9123 unsigned Lane =
Op.getConstantOperandVal(1);
9124 unsigned LaneWidth =
9135 if (!isa<ConstantSDNode>(Lane))
9155 assert(
Op.getValueType().getScalarSizeInBits() == 1 &&
9156 "Unexpected custom CONCAT_VECTORS lowering");
9158 "Unexpected custom CONCAT_VECTORS lowering");
9159 assert(ST->hasMVEIntegerOps() &&
9160 "CONCAT_VECTORS lowering only supported for MVE");
9164 EVT Op2VT = V2.getValueType();
9165 assert(Op1VT == Op2VT &&
"Operand types don't match!");
9166 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
9167 "Unexpected i1 concat operations!");
9180 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
9194 auto ExtractInto = [&DAG, &dl](
SDValue NewV,
SDValue ConVec,
unsigned &j) {
9195 EVT NewVT = NewV.getValueType();
9196 EVT ConcatVT = ConVec.getValueType();
9197 unsigned ExtScale = 1;
9198 if (NewVT == MVT::v2f64) {
9212 ConVec = ExtractInto(NewV1, ConVec, j);
9213 ConVec = ExtractInto(NewV2, ConVec, j);
9223 while (ConcatOps.
size() > 1) {
9224 for (
unsigned I = 0, E = ConcatOps.
size();
I != E;
I += 2) {
9227 ConcatOps[
I / 2] = ConcatPair(V1, V2);
9231 return ConcatOps[0];
9236 EVT VT =
Op->getValueType(0);
9243 "unexpected CONCAT_VECTORS");
9264 EVT VT =
Op.getValueType();
9267 unsigned Index = V2->getAsZExtVal();
9270 "Unexpected custom EXTRACT_SUBVECTOR lowering");
9271 assert(ST->hasMVEIntegerOps() &&
9272 "EXTRACT_SUBVECTOR lowering only supported for MVE");
9282 EVT SubVT = MVT::v4i32;
9284 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j += 2) {
9299 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j++) {
9315 assert(ST->hasMVEIntegerOps() &&
"Expected MVE!");
9316 EVT VT =
N->getValueType(0);
9317 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9318 "Expected a vector i1 type!");
9320 EVT FromVT =
Op.getValueType();
9331 if (!Subtarget->hasMVEIntegerOps())
9334 EVT ToVT =
N->getValueType(0);
9377 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9379 EVT FromVT =
N->getOperand(0).getValueType();
9380 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9391 if (!Subtarget->hasMVEIntegerOps())
9396 EVT ToVT =
N->getValueType(0);
9397 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9400 EVT FromVT =
Op.getValueType();
9401 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9412 SDValue Ext1 = Ext.getValue(1);
9415 Ext = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext);
9416 Ext1 = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext1);
9428 EVT VT =
N->getValueType(0);
9430 SDNode *BVN =
N->getOperand(0).getNode();
9435 unsigned HiElt = 1 - LoElt;
9440 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9456 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
9457 SDNode *Elt =
N->getOperand(i).getNode();
9460 unsigned HalfSize = EltSize / 2;
9462 if (!
isIntN(HalfSize,
C->getSExtValue()))
9465 if (!
isUIntN(HalfSize,
C->getZExtValue()))
9504 switch (OrigSimpleTy) {
9520 unsigned ExtOpcode) {
9543 if (ExtendedTy == LD->getMemoryVT())
9544 return DAG.
getLoad(LD->getMemoryVT(),
SDLoc(LD), LD->getChain(),
9545 LD->getBasePtr(), LD->getPointerInfo(), LD->getAlign(),
9546 LD->getMemOperand()->getFlags());
9552 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
9553 LD->getMemoryVT(), LD->getAlign(),
9554 LD->getMemOperand()->getFlags());
9567 N->getOperand(0)->getValueType(0),
9573 "Expected extending load");
9579 DAG.
getNode(Opcode,
SDLoc(newLoad), LD->getValueType(0), newLoad);
9588 SDNode *BVN =
N->getOperand(0).getNode();
9590 BVN->
getValueType(0) == MVT::v4i32 &&
"expected v4i32 BUILD_VECTOR");
9598 EVT VT =
N->getValueType(0);
9604 for (
unsigned i = 0; i != NumElts; ++i) {
9605 const APInt &CInt =
N->getConstantOperandAPInt(i);
9614 unsigned Opcode =
N->getOpcode();
9616 SDNode *N0 =
N->getOperand(0).getNode();
9617 SDNode *N1 =
N->getOperand(1).getNode();
9625 unsigned Opcode =
N->getOpcode();
9627 SDNode *N0 =
N->getOperand(0).getNode();
9628 SDNode *N1 =
N->getOperand(1).getNode();
9638 EVT VT =
Op.getValueType();
9640 "unexpected type for custom-lowering ISD::MUL");
9641 SDNode *N0 =
Op.getOperand(0).getNode();
9642 SDNode *N1 =
Op.getOperand(1).getNode();
9643 unsigned NewOpc = 0;
9647 if (isN0SExt && isN1SExt)
9652 if (isN0ZExt && isN1ZExt)
9654 else if (isN1SExt || isN1ZExt) {
9671 if (VT == MVT::v2i64)
9688 "unexpected types for extended operands to VMULL");
9689 return DAG.
getNode(NewOpc,
DL, VT, Op0, Op1);
9724 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9758 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9761 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9782 EVT VT =
Op.getValueType();
9783 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9784 "unexpected type for custom-lowering ISD::SDIV");
9791 if (VT == MVT::v8i8) {
9819 EVT VT =
Op.getValueType();
9820 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9821 "unexpected type for custom-lowering ISD::UDIV");
9828 if (VT == MVT::v8i8) {
9867 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9870 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9874 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9895 EVT VT =
N->getValueType(0);
9909 Op.getOperand(1), Carry);
9923 Op.getOperand(1), Carry);
9956 bool ShouldUseSRet = Subtarget->
isAPCS_ABI();
9958 if (ShouldUseSRet) {
9968 Entry.IsSExt =
false;
9969 Entry.IsZExt =
false;
9970 Entry.IsSRet =
true;
9971 Args.push_back(Entry);
9978 Entry.IsSExt =
false;
9979 Entry.IsZExt =
false;
9980 Args.push_back(Entry);
9983 (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
9993 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
9996 return CallResult.first;
10015 EVT VT =
Op.getValueType();
10016 assert((VT == MVT::i32 || VT == MVT::i64) &&
10017 "unexpected type for custom lowering DIV");
10023 const char *
Name =
nullptr;
10025 Name = (VT == MVT::i32) ?
"__rt_sdiv" :
"__rt_sdiv64";
10027 Name = (VT == MVT::i32) ?
"__rt_udiv" :
"__rt_udiv64";
10033 for (
auto AI : {1, 0}) {
10035 Arg.Node =
Op.getOperand(AI);
10036 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.
getContext());
10037 Args.push_back(Arg);
10040 CallLoweringInfo CLI(DAG);
10044 ES, std::move(Args));
10054ARMTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
10062 const bool MinSize =
ST.hasMinSize();
10063 const bool HasDivide =
ST.isThumb() ?
ST.hasDivideInThumbMode()
10064 :
ST.hasDivideInARMMode();
10068 if (
N->getOperand(0).getValueType().isVector())
10073 if (!(MinSize && HasDivide))
10086 if (Divisor.
sgt(128))
10094 assert(
Op.getValueType() == MVT::i32 &&
10095 "unexpected type for custom lowering DIV");
10101 return LowerWindowsDIVLibCall(
Op, DAG,
Signed, DBZCHK);
10107 if (
N->getValueType(0) == MVT::i32)
10115void ARMTargetLowering::ExpandDIV_Windows(
10121 assert(
Op.getValueType() == MVT::i64 &&
10122 "unexpected type for custom lowering DIV");
10139 EVT MemVT = LD->getMemoryVT();
10140 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10141 MemVT == MVT::v16i1) &&
10142 "Expected a predicate type!");
10143 assert(MemVT ==
Op.getValueType());
10145 "Expected a non-extending load");
10146 assert(LD->isUnindexed() &&
"Expected a unindexed load");
10160 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(),
10162 LD->getMemOperand());
10169 if (MemVT != MVT::v16i1)
10178 EVT MemVT =
LD->getMemoryVT();
10179 assert(
LD->isUnindexed() &&
"Loads should be unindexed at this point.");
10181 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10187 {LD->getChain(), LD->getBasePtr()}, MemVT,
LD->getMemOperand());
10197 EVT MemVT = ST->getMemoryVT();
10198 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10199 MemVT == MVT::v16i1) &&
10200 "Expected a predicate type!");
10201 assert(MemVT == ST->getValue().getValueType());
10202 assert(!ST->isTruncatingStore() &&
"Expected a non-extending store");
10203 assert(ST->isUnindexed() &&
"Expected a unindexed store");
10208 SDValue Build = ST->getValue();
10209 if (MemVT != MVT::v16i1) {
10228 ST->getChain(), dl, GRP, ST->getBasePtr(),
10230 ST->getMemOperand());
10236 EVT MemVT = ST->getMemoryVT();
10237 assert(ST->isUnindexed() &&
"Stores should be unindexed at this point.");
10239 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10255 {ST->getChain(), Lo, Hi, ST->getBasePtr()},
10256 MemVT, ST->getMemOperand());
10257 }
else if (Subtarget->hasMVEIntegerOps() &&
10258 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10259 MemVT == MVT::v16i1))) {
10274 MVT VT =
Op.getSimpleValueType();
10276 SDValue PassThru =
N->getPassThru();
10287 VT, dl,
N->getChain(),
N->getBasePtr(),
N->getOffset(), Mask, ZeroVec,
10288 N->getMemoryVT(),
N->getMemOperand(),
N->getAddressingMode(),
10289 N->getExtensionType(),
N->isExpandingLoad());
10294 if (!PassThru.
isUndef() && !PassThruIsCastZero)
10301 if (!ST->hasMVEIntegerOps())
10305 unsigned BaseOpcode = 0;
10306 switch (
Op->getOpcode()) {
10322 unsigned NumActiveLanes = NumElts;
10324 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10325 NumActiveLanes == 2) &&
10326 "Only expected a power 2 vector size");
10330 while (NumActiveLanes > 4) {
10333 Op0 = DAG.
getNode(BaseOpcode, dl, VT, Op0, Rev);
10334 NumActiveLanes /= 2;
10338 if (NumActiveLanes == 4) {
10348 SDValue Res0 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10349 SDValue Res1 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext2, Ext3,
Op->getFlags());
10350 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res0, Res1,
Op->getFlags());
10356 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10360 if (EltVT !=
Op->getValueType(0))
10367 if (!ST->hasMVEFloatOps())
10374 if (!ST->hasNEON())
10382 unsigned PairwiseIntrinsic = 0;
10383 switch (
Op->getOpcode()) {
10387 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
10390 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
10393 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
10396 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10402 unsigned NumActiveLanes = NumElts;
10404 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10405 NumActiveLanes == 2) &&
10406 "Only expected a power 2 vector size");
10412 VT =
Lo.getValueType();
10414 NumActiveLanes /= 2;
10418 while (NumActiveLanes > 1) {
10420 NumActiveLanes /= 2;
10427 if (EltVT !=
Op.getValueType()) {
10428 unsigned Extend = 0;
10429 switch (
Op->getOpcode()) {
10441 Res = DAG.
getNode(Extend, dl,
Op.getValueType(), Res);
10463 SDValue Ops[] = {
N->getOperand(0),
10473 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
10480 SDLoc dl(V.getNode());
10481 auto [VLo, VHi] = DAG.
SplitScalar(V, dl, MVT::i32, MVT::i32);
10489 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
10491 DAG.
getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
10497 assert(
N->getValueType(0) == MVT::i64 &&
10498 "AtomicCmpSwap on types less than 64 should be legal");
10499 SDValue Ops[] = {
N->getOperand(1),
10504 ARM::CMP_SWAP_64,
SDLoc(
N),
10505 DAG.
getVTList(MVT::Untyped, MVT::i32, MVT::Other), Ops);
10524 EVT VT =
Op.getValueType();
10533 if (isUnsupportedFloatingType(
LHS.getValueType())) {
10535 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS, Chain, IsSignaling);
10536 if (!
RHS.getNode()) {
10557 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10558 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, CCR, Cmp, DAG);
10560 ARMcc = DAG.
getConstant(CondCode2, dl, MVT::i32);
10561 Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10562 Result = getCMOV(dl, VT, Result, True, ARMcc, CCR, Cmp, DAG);
10578 switch (
Op.getOpcode()) {
10610 case ISD::BITCAST:
return ExpandBITCAST(
Op.getNode(), DAG, Subtarget);
10614 case ISD::SREM:
return LowerREM(
Op.getNode(), DAG);
10615 case ISD::UREM:
return LowerREM(
Op.getNode(), DAG);
10637 return LowerSET_FPMODE(
Op, DAG);
10639 return LowerRESET_FPMODE(
Op, DAG);
10643 return LowerDIV_Windows(
Op, DAG,
true);
10647 return LowerDIV_Windows(
Op, DAG,
false);
10654 return LowerSignedALUO(
Op, DAG);
10657 return LowerUnsignedALUO(
Op, DAG);
10691 return LowerDYNAMIC_STACKALLOC(
Op, DAG);
10700 return LowerSPONENTRY(
Op, DAG);
10707 unsigned IntNo =
N->getConstantOperandVal(0);
10709 if (IntNo == Intrinsic::arm_smlald)
10711 else if (IntNo == Intrinsic::arm_smlaldx)
10713 else if (IntNo == Intrinsic::arm_smlsld)
10715 else if (IntNo == Intrinsic::arm_smlsldx)
10722 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(3), dl, MVT::i32, MVT::i32);
10726 N->getOperand(1),
N->getOperand(2),
10738 switch (
N->getOpcode()) {
10745 Res = ExpandBITCAST(
N, DAG, Subtarget);
10754 Res = LowerREM(
N, DAG);
10758 Res = LowerDivRem(
SDValue(
N, 0), DAG);
10812 "ROPI/RWPI not currently supported with SjLj");
10821 bool isThumb = Subtarget->isThumb();
10822 bool isThumb2 = Subtarget->
isThumb2();
10825 unsigned PCAdj = (
isThumb || isThumb2) ? 4 : 8;
10831 : &ARM::GPRRegClass;
10849 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10855 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10861 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10879 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10884 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10889 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10894 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
10900 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
10915 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10921 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10945 : &ARM::GPRnopcRegClass;
10950 unsigned MaxCSNum = 0;
10958 if (!
II.isEHLabel())
10962 if (!MF->hasCallSiteLandingPad(
Sym))
continue;
10965 for (
unsigned Idx : CallSiteIdxs) {
10966 CallSiteNumToLPad[
Idx].push_back(&BB);
10967 MaxCSNum = std::max(MaxCSNum,
Idx);
10974 std::vector<MachineBasicBlock*> LPadList;
10977 for (
unsigned I = 1;
I <= MaxCSNum; ++
I) {
10980 LPadList.push_back(
MBB);
10985 assert(!LPadList.empty() &&
10986 "No landing pad destinations for the dispatch jump table!");
11000 unsigned trap_opcode;
11001 if (Subtarget->isThumb())
11002 trap_opcode = ARM::tTRAP;
11004 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP;
11013 MF->insert(MF->end(), DispatchBB);
11014 MF->insert(MF->end(), DispContBB);
11015 MF->insert(MF->end(), TrapBB);
11019 SetupEntryBlockForSjLj(
MI,
MBB, DispatchBB, FI);
11026 MIB =
BuildMI(DispatchBB, dl,
TII->get(ARM::Int_eh_sjlj_dispatchsetup));
11038 unsigned NumLPads = LPadList.size();
11040 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11041 BuildMI(DispatchBB, dl,
TII->get(ARM::t2LDRi12), NewVReg1)
11047 if (NumLPads < 256) {
11048 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPri))
11050 .
addImm(LPadList.size())
11053 Register VReg1 =
MRI->createVirtualRegister(TRC);
11054 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVi16), VReg1)
11055 .
addImm(NumLPads & 0xFFFF)
11058 unsigned VReg2 = VReg1;
11059 if ((NumLPads & 0xFFFF0000) != 0) {
11060 VReg2 =
MRI->createVirtualRegister(TRC);
11061 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVTi16), VReg2)
11067 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPrr))
11073 BuildMI(DispatchBB, dl,
TII->get(ARM::t2Bcc))
11078 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11079 BuildMI(DispContBB, dl,
TII->get(ARM::t2LEApcrelJT), NewVReg3)
11083 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11084 BuildMI(DispContBB, dl,
TII->get(ARM::t2ADDrs), NewVReg4)
11091 BuildMI(DispContBB, dl,
TII->get(ARM::t2BR_JT))
11095 }
else if (Subtarget->isThumb()) {
11096 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11097 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRspi), NewVReg1)
11103 if (NumLPads < 256) {
11104 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPi8))
11111 const Constant *
C = ConstantInt::get(Int32Ty, NumLPads);
11114 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
11117 Register VReg1 =
MRI->createVirtualRegister(TRC);
11118 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRpci))
11122 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPr))
11128 BuildMI(DispatchBB, dl,
TII->get(ARM::tBcc))
11133 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
11134 BuildMI(DispContBB, dl,
TII->get(ARM::tLSLri), NewVReg2)
11140 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11141 BuildMI(DispContBB, dl,
TII->get(ARM::tLEApcrelJT), NewVReg3)
11145 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11146 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg4)
11156 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11157 BuildMI(DispContBB, dl,
TII->get(ARM::tLDRi), NewVReg5)
11163 unsigned NewVReg6 = NewVReg5;
11164 if (IsPositionIndependent) {
11165 NewVReg6 =
MRI->createVirtualRegister(TRC);
11166 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg6)
11173 BuildMI(DispContBB, dl,
TII->get(ARM::tBR_JTr))
11177 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11178 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRi12), NewVReg1)
11184 if (NumLPads < 256) {
11185 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPri))
11189 }
else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) {
11190 Register VReg1 =
MRI->createVirtualRegister(TRC);
11191 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVi16), VReg1)
11192 .
addImm(NumLPads & 0xFFFF)
11195 unsigned VReg2 = VReg1;
11196 if ((NumLPads & 0xFFFF0000) != 0) {
11197 VReg2 =
MRI->createVirtualRegister(TRC);
11198 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVTi16), VReg2)
11204 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11211 const Constant *
C = ConstantInt::get(Int32Ty, NumLPads);
11214 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
11217 Register VReg1 =
MRI->createVirtualRegister(TRC);
11218 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRcp))
11223 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11234 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11235 BuildMI(DispContBB, dl,
TII->get(ARM::MOVsi), NewVReg3)
11240 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11241 BuildMI(DispContBB, dl,
TII->get(ARM::LEApcrelJT), NewVReg4)
11248 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11249 BuildMI(DispContBB, dl,
TII->get(ARM::LDRrs), NewVReg5)
11256 if (IsPositionIndependent) {
11257 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTadd))
11262 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTr))
11271 if (SeenMBBs.
insert(CurMBB).second)
11283 while (!Successors.empty()) {
11292 BB->normalizeSuccProbs();
11299 II = BB->rbegin(), IE = BB->rend();
II != IE; ++
II) {
11300 if (!
II->isCall())
continue;
11304 OI =
II->operands_begin(), OE =
II->operands_end();
11306 if (!OI->isReg())
continue;
11307 DefRegs[OI->getReg()] =
true;
11312 for (
unsigned i = 0; SavedRegs[i] != 0; ++i) {
11313 unsigned Reg = SavedRegs[i];
11315 !ARM::tGPRRegClass.contains(Reg) &&
11316 !ARM::hGPRRegClass.contains(Reg))
11318 if (Subtarget->
isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11320 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11333 MBBLPad->setIsEHPad(
false);
11336 MI.eraseFromParent();
11349static unsigned getLdOpcode(
unsigned LdSize,
bool IsThumb1,
bool IsThumb2) {
11351 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11352 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11354 return LdSize == 4 ? ARM::tLDRi
11355 : LdSize == 2 ? ARM::tLDRHi
11356 : LdSize == 1 ? ARM::tLDRBi : 0;
11358 return LdSize == 4 ? ARM::t2LDR_POST
11359 : LdSize == 2 ? ARM::t2LDRH_POST
11360 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11361 return LdSize == 4 ? ARM::LDR_POST_IMM
11362 : LdSize == 2 ? ARM::LDRH_POST
11363 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11368static unsigned getStOpcode(
unsigned StSize,
bool IsThumb1,
bool IsThumb2) {
11370 return StSize == 16 ? ARM::VST1q32wb_fixed
11371 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11373 return StSize == 4 ? ARM::tSTRi
11374 : StSize == 2 ? ARM::tSTRHi
11375 : StSize == 1 ? ARM::tSTRBi : 0;
11377 return StSize == 4 ? ARM::t2STR_POST
11378 : StSize == 2 ? ARM::t2STRH_POST
11379 : StSize == 1 ? ARM::t2STRB_POST : 0;
11380 return StSize == 4 ? ARM::STR_POST_IMM
11381 : StSize == 2 ? ARM::STRH_POST
11382 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11389 unsigned LdSize,
unsigned Data,
unsigned AddrIn,
11390 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11391 unsigned LdOpc =
getLdOpcode(LdSize, IsThumb1, IsThumb2);
11392 assert(LdOpc != 0 &&
"Should have a load opcode");
11399 }
else if (IsThumb1) {
11405 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11410 }
else if (IsThumb2) {
11430 unsigned StSize,
unsigned Data,
unsigned AddrIn,
11431 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11432 unsigned StOpc =
getStOpcode(StSize, IsThumb1, IsThumb2);
11433 assert(StOpc != 0 &&
"Should have a store opcode");
11435 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11440 }
else if (IsThumb1) {
11447 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11452 }
else if (IsThumb2) {
11453 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11459 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11480 unsigned SizeVal =
MI.getOperand(2).getImm();
11481 unsigned Alignment =
MI.getOperand(3).getImm();
11486 unsigned UnitSize = 0;
11491 bool IsThumb2 = Subtarget->
isThumb2();
11492 bool IsThumb = Subtarget->isThumb();
11494 if (Alignment & 1) {
11496 }
else if (Alignment & 2) {
11501 Subtarget->hasNEON()) {
11502 if ((Alignment % 16 == 0) && SizeVal >= 16)
11504 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11513 bool IsNeon = UnitSize >= 8;
11514 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11516 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11517 : UnitSize == 8 ? &ARM::DPRRegClass
11520 unsigned BytesLeft = SizeVal % UnitSize;
11521 unsigned LoopSize = SizeVal - BytesLeft;
11523 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11527 unsigned srcIn = src;
11528 unsigned destIn = dest;
11529 for (
unsigned i = 0; i < LoopSize; i+=UnitSize) {
11530 Register srcOut =
MRI.createVirtualRegister(TRC);
11531 Register destOut =
MRI.createVirtualRegister(TRC);
11532 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11534 IsThumb1, IsThumb2);
11536 IsThumb1, IsThumb2);
11544 for (
unsigned i = 0; i < BytesLeft; i++) {
11545 Register srcOut =
MRI.createVirtualRegister(TRC);
11546 Register destOut =
MRI.createVirtualRegister(TRC);
11547 Register scratch =
MRI.createVirtualRegister(TRC);
11549 IsThumb1, IsThumb2);
11551 IsThumb1, IsThumb2);
11555 MI.eraseFromParent();
11581 MF->
insert(It, loopMBB);
11582 MF->
insert(It, exitMBB);
11585 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
11595 Register varEnd =
MRI.createVirtualRegister(TRC);
11597 BuildMI(BB, dl,
TII->get(IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11600 }
else if (Subtarget->genExecuteOnly()) {
11601 assert(IsThumb &&
"Non-thumb expected to have used movt");
11606 const Constant *
C = ConstantInt::get(Int32Ty, LoopSize);
11637 Register varLoop =
MRI.createVirtualRegister(TRC);
11638 Register varPhi =
MRI.createVirtualRegister(TRC);
11639 Register srcLoop =
MRI.createVirtualRegister(TRC);
11640 Register srcPhi =
MRI.createVirtualRegister(TRC);
11641 Register destLoop =
MRI.createVirtualRegister(TRC);
11642 Register destPhi =
MRI.createVirtualRegister(TRC);
11650 BuildMI(BB, dl,
TII->get(ARM::PHI), destPhi)
11656 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11658 IsThumb1, IsThumb2);
11660 IsThumb1, IsThumb2);
11664 BuildMI(*BB, BB->
end(), dl,
TII->get(ARM::tSUBi8), varLoop)
11672 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
11681 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11690 auto StartOfExit = exitMBB->
begin();
11694 unsigned srcIn = srcLoop;
11695 unsigned destIn = destLoop;
11696 for (
unsigned i = 0; i < BytesLeft; i++) {
11697 Register srcOut =
MRI.createVirtualRegister(TRC);
11698 Register destOut =
MRI.createVirtualRegister(TRC);
11699 Register scratch =
MRI.createVirtualRegister(TRC);
11700 emitPostLd(BB, StartOfExit,
TII, dl, 1, scratch, srcIn, srcOut,
11701 IsThumb1, IsThumb2);
11702 emitPostSt(BB, StartOfExit,
TII, dl, 1, scratch, destIn, destOut,
11703 IsThumb1, IsThumb2);
11708 MI.eraseFromParent();
11720 "__chkstk is only supported on Windows");
11721 assert(Subtarget->
isThumb2() &&
"Windows on ARM requires Thumb-2 mode");
11741 switch (
TM.getCodeModel()) {
11783 MI.eraseFromParent();
11807 .
addReg(
MI.getOperand(0).getReg())
11815 MI.eraseFromParent();
11839 if (miI == BB->
end()) {
11841 if (Succ->isLiveIn(ARM::CPSR))
11847 SelectItr->addRegisterKilled(ARM::CPSR,
TRI);
11859 Register AddDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11860 BuildMI(TpEntry, Dl,
TII->get(ARM::t2ADDri), AddDestReg)
11866 Register LsrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11867 BuildMI(TpEntry, Dl,
TII->get(ARM::t2LSRri), LsrDestReg)
11873 Register TotalIterationsReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11874 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg)
11877 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopStart))
11878 .
addUse(TotalIterationsReg)
11885 return TotalIterationsReg;
11896 Register TotalIterationsReg,
bool IsMemcpy) {
11903 SrcPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11904 CurrSrcReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11905 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), SrcPhiReg)
11913 Register DestPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11914 Register CurrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11915 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), DestPhiReg)
11922 Register LoopCounterPhiReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11923 Register RemainingLoopIterationsReg =
11924 MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11925 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), LoopCounterPhiReg)
11926 .
addUse(TotalIterationsReg)
11928 .
addUse(RemainingLoopIterationsReg)
11932 Register PredCounterPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11933 Register RemainingElementsReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11934 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), PredCounterPhiReg)
11935 .
addUse(ElementCountReg)
11937 .
addUse(RemainingElementsReg)
11941 Register VccrReg =
MRI.createVirtualRegister(&ARM::VCCRRegClass);
11942 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VCTP8), VccrReg)
11943 .
addUse(PredCounterPhiReg)
11948 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2SUBri), RemainingElementsReg)
11949 .
addUse(PredCounterPhiReg)
11957 SrcValueReg =
MRI.createVirtualRegister(&ARM::MQPRRegClass);
11958 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VLDRBU8_post))
11967 SrcValueReg = OpSrcReg;
11969 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VSTRBU8_post))
11980 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg)
11981 .
addUse(LoopCounterPhiReg)
11984 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopEnd))
11985 .
addUse(RemainingLoopIterationsReg)
11998 bool isThumb2 = Subtarget->
isThumb2();
11999 switch (
MI.getOpcode()) {
12006 case ARM::tLDR_postidx: {
12010 .
add(
MI.getOperand(2))
12011 .
add(
MI.getOperand(3))
12012 .
add(
MI.getOperand(4))
12013 .
add(
MI.getOperand(0))
12015 MI.eraseFromParent();
12019 case ARM::MVE_MEMCPYLOOPINST:
12020 case ARM::MVE_MEMSETLOOPINST: {
12050 Register OpDestReg =
MI.getOperand(0).getReg();
12051 Register OpSrcReg =
MI.getOperand(1).getReg();
12052 Register OpSizeReg =
MI.getOperand(2).getReg();
12072 if (TpExit == BB) {
12074 "block containing memcpy/memset Pseudo");
12087 bool IsMemcpy =
MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
12089 OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy);
12104 MI.eraseFromParent();
12114 case ARM::t2STR_preidx:
12115 MI.setDesc(
TII->get(ARM::t2STR_PRE));
12117 case ARM::t2STRB_preidx:
12118 MI.setDesc(
TII->get(ARM::t2STRB_PRE));
12120 case ARM::t2STRH_preidx:
12121 MI.setDesc(
TII->get(ARM::t2STRH_PRE));
12124 case ARM::STRi_preidx:
12125 case ARM::STRBi_preidx: {
12126 unsigned NewOpc =
MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
12127 : ARM::STRB_PRE_IMM;
12129 unsigned Offset =
MI.getOperand(4).getImm();
12137 .
add(
MI.getOperand(0))
12138 .
add(
MI.getOperand(1))
12139 .
add(
MI.getOperand(2))
12141 .
add(
MI.getOperand(5))
12142 .
add(
MI.getOperand(6))
12144 MI.eraseFromParent();
12147 case ARM::STRr_preidx:
12148 case ARM::STRBr_preidx:
12149 case ARM::STRH_preidx: {
12151 switch (
MI.getOpcode()) {
12153 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG;
break;
12154 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG;
break;
12155 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE;
break;
12160 MI.eraseFromParent();
12164 case ARM::tMOVCCr_pseudo: {
12182 F->insert(It, copy0MBB);
12183 F->insert(It, sinkMBB);
12186 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
12192 if (!
MI.killsRegister(ARM::CPSR,
nullptr) &&
12208 .
addImm(
MI.getOperand(3).getImm())
12209 .
addReg(
MI.getOperand(4).getReg());
12224 .
addReg(
MI.getOperand(1).getReg())
12226 .
addReg(
MI.getOperand(2).getReg())
12229 MI.eraseFromParent();
12234 case ARM::BCCZi64: {
12240 bool RHSisZero =
MI.getOpcode() == ARM::BCCZi64;
12245 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12249 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12255 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12259 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12269 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12278 MI.eraseFromParent();
12282 case ARM::Int_eh_sjlj_setjmp:
12283 case ARM::Int_eh_sjlj_setjmp_nofp:
12284 case ARM::tInt_eh_sjlj_setjmp:
12285 case ARM::t2Int_eh_sjlj_setjmp:
12286 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12289 case ARM::Int_eh_sjlj_setup_dispatch:
12290 EmitSjLjDispatchBlock(
MI, BB);
12313 Fn->
insert(BBI, SinkBB);
12315 Register ABSSrcReg =
MI.getOperand(1).getReg();
12316 Register ABSDstReg =
MI.getOperand(0).getReg();
12317 bool ABSSrcKIll =
MI.getOperand(1).isKill();
12318 bool isThumb2 = Subtarget->
isThumb2();
12322 Register NewRsbDstReg =
MRI.createVirtualRegister(
12323 isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass);
12337 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12344 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).
addMBB(SinkBB)
12351 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
12360 TII->get(ARM::PHI), ABSDstReg)
12365 MI.eraseFromParent();
12370 case ARM::COPY_STRUCT_BYVAL_I32:
12372 return EmitStructByval(
MI, BB);
12373 case ARM::WIN__CHKSTK:
12374 return EmitLowered__chkstk(
MI, BB);
12375 case ARM::WIN__DBZCHK:
12376 return EmitLowered__dbzchk(
MI, BB);
12393 if (!
Node->hasAnyUseOfValue(0)) {
12394 MI.getOperand(0).setIsDead(
true);
12396 if (!
Node->hasAnyUseOfValue(1)) {
12397 MI.getOperand(1).setIsDead(
true);
12401 for (
unsigned I = 0;
I !=
MI.getOperand(4).getImm(); ++
I) {
12402 Register TmpReg =
MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
12403 : &ARM::GPRRegClass);
12410 if (
MI.getOpcode() == ARM::MEMCPY) {
12428 MCID = &
TII->get(NewOpc);
12431 MI.getDesc().getNumOperands() + 5 -
MI.getDesc().getSize()
12432 &&
"converted opcode should be the same except for cc_out"
12433 " (and, on Thumb1, pred)");
12443 MI.addOperand(
MI.getOperand(1));
12444 MI.removeOperand(1);
12448 for (
unsigned i =
MI.getNumOperands(); i--;) {
12450 if (
op.isReg() &&
op.isUse()) {
12453 MI.tieOperands(DefIdx, i);
12467 if (!
MI.hasOptionalDef() || !MCID->
operands()[ccOutIdx].isOptionalDef()) {
12468 assert(!NewOpc &&
"Optional cc_out operand required");
12473 bool definesCPSR =
false;
12474 bool deadCPSR =
false;
12475 for (
unsigned i = MCID->
getNumOperands(), e =
MI.getNumOperands(); i != e;
12479 definesCPSR =
true;
12482 MI.removeOperand(i);
12486 if (!definesCPSR) {
12487 assert(!NewOpc &&
"Optional cc_out operand required");
12490 assert(deadCPSR == !Node->hasAnyUseOfValue(1) &&
"inconsistent dead flag");
12492 assert(!
MI.getOperand(ccOutIdx).getReg() &&
12493 "expect uninitialized optional cc_out operand");
12531 switch (
N->getOpcode()) {
12532 default:
return false;
12534 CC =
N->getOperand(0);
12556 EVT VT =
N->getValueType(0);
12557 CC =
N->getOperand(0);
12558 if (
CC.getValueType() != MVT::i1 ||
CC.getOpcode() !=
ISD::SETCC)
12604 EVT VT =
N->getValueType(0);
12607 bool SwapSelectOps;
12609 NonConstantVal, DAG))
12615 OtherOp, NonConstantVal);
12621 CCOp, TrueVal, FalseVal);
12645 if (
N->getOpcode() ==
ARMISD::VTRN &&
N->getValueType(0) == MVT::v2i32)
12660 if (!
N->getValueType(0).is64BitVector())
12668 EVT VT =
N->getValueType(0);
12707 EVT VT =
N->getValueType(0);
12713 Opcode = Intrinsic::arm_neon_vpaddls;
12715 Opcode = Intrinsic::arm_neon_vpaddlu;
12743 EVT VT =
N->getValueType(0);
12758 unsigned nextIndex = 0;
12826 return DAG.
getNode(ExtOp, dl, VT, tmp);
12857 if (SRA.getOpcode() !=
ISD::SRA) {
12863 if (
auto Const = dyn_cast<ConstantSDNode>(SRA.getOperand(1))) {
12864 if (Const->getZExtValue() != 31)
12869 if (SRA.getOperand(0) !=
Mul)
12873 SDLoc dl(AddcNode);
12874 unsigned Opcode = 0;
12880 Op0 =
Mul.getOperand(0);
12881 Op1 =
Mul.getOperand(1);
12884 Op0 =
Mul.getOperand(0);
12885 Op1 =
Mul.getOperand(1).getOperand(0);
12888 Op0 =
Mul.getOperand(0).getOperand(0);
12889 Op1 =
Mul.getOperand(1);
12892 Op0 =
Mul->getOperand(0).getOperand(0);
12893 Op1 =
Mul->getOperand(1).getOperand(0);
12902 SDValue HiMLALResult(SMLAL.getNode(), 1);
12903 SDValue LoMLALResult(SMLAL.getNode(), 0);
12909 SDValue resNode(AddcNode, 0);
12938 "Expect an ADDE or SUBE");
12942 "ADDE node has the wrong inputs");
12961 "Expect ADDC with two result values. First: i32");
12981 bool IsLeftOperandMUL =
false;
12986 IsLeftOperandMUL =
true;
12997 SDValue *LowAddSub =
nullptr;
13000 if ((AddeSubeOp0 != MULOp.
getValue(1)) && (AddeSubeOp1 != MULOp.
getValue(1)))
13003 if (IsLeftOperandMUL)
13004 HiAddSub = &AddeSubeOp1;
13006 HiAddSub = &AddeSubeOp0;
13011 if (AddcSubcOp0 == MULOp.
getValue(0)) {
13012 LoMul = &AddcSubcOp0;
13013 LowAddSub = &AddcSubcOp1;
13015 if (AddcSubcOp1 == MULOp.
getValue(0)) {
13016 LoMul = &AddcSubcOp1;
13017 LowAddSub = &AddcSubcOp0;
13025 if (AddcSubcNode == HiAddSub->getNode() ||
13041 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->
useMulOps() &&
13055 return SDValue(AddeSubeNode, 0);
13066 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13076 return SDValue(AddeSubeNode, 0);
13088 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13097 SDNode *UmlalNode =
nullptr;
13122 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13136 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13141 SDNode* AddcNode =
N->getOperand(2).getNode();
13142 SDNode* AddeNode =
N->getOperand(3).getNode();
13150 {N->getOperand(0), N->getOperand(1),
13151 AddcNode->getOperand(0), AddcNode->getOperand(1)});
13175 int32_t imm =
C->getSExtValue();
13176 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13181 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
N->getOperand(0),
RHS);
13196 int64_t imm =
C->getSExtValue();
13207 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
13208 N->getOperand(0),
RHS,
N->getOperand(2));
13220 if (!Subtarget->hasMVEIntegerOps())
13233 SetCC =
N->getOperand(0);
13236 CC = cast<CondCodeSDNode>(SetCC->
getOperand(2))->get();
13237 TrueVal =
N->getOperand(1);
13238 FalseVal =
N->getOperand(2);
13240 LHS =
N->getOperand(0);
13241 RHS =
N->getOperand(1);
13242 CC = cast<CondCodeSDNode>(
N->getOperand(4))->get();
13243 TrueVal =
N->getOperand(2);
13244 FalseVal =
N->getOperand(3);
13249 unsigned int Opcode = 0;
13278 switch (TrueVal->getOpcode()) {
13297 if (TrueVal !=
LHS || FalseVal !=
RHS)
13300 EVT LeftType =
LHS->getValueType(0);
13301 EVT RightType =
RHS->getValueType(0);
13304 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13308 if (VectorScalarType != MVT::i32)
13316 if (VectorScalarType != MVT::i32)
13329 EVT VT =
N->getValueType(0);
13337 Shft =
N->getOperand(0);
13343 cast<CondCodeSDNode>(Cmp.getOperand(2))->get() !=
ISD::SETLT ||
13344 Cmp.getOperand(0) !=
N->getOperand(1) ||
13345 Cmp.getOperand(1) !=
N->getOperand(2))
13347 Shft =
N->getOperand(1);
13359 ScalarType = MVT::i8;
13362 case (1 << 15) - 1:
13363 ScalarType = MVT::i16;
13366 case (1ULL << 31) - 1:
13367 ScalarType = MVT::i32;
13398 unsigned LegalLanes = 128 / (ShftAmt + 1);
13422 for (
unsigned I = 0;
I < NumParts; ++
I) {
13439 if (!Subtarget->hasMVEIntegerOps())
13454 if (
N->getOperand(0).getOpcode() !=
ISD::XOR)
13464 if (!Const || !Const->isOne())
13482 EVT VT =
N->getValueType(0);
13484 if (!Subtarget->hasMVEIntegerOps() ||
13513 Opc = Intrinsic::arm_mve_vctp64;
13516 Opc = Intrinsic::arm_mve_vctp32;
13519 Opc = Intrinsic::arm_mve_vctp16;
13522 Opc = Intrinsic::arm_mve_vctp8;
13576 EVT VT =
N->getValueType(0);
13582 switch (
Op.getOpcode()) {
13599 !isa<ConstantSDNode>(N0) && N1->
hasOneUse()) {
13607 unsigned N0RedOp = 0;
13614 unsigned N1RedOp = 0;
13628 if (
SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13630 if (
SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13637 auto DistrubuteVecReduceLoad = [&](
SDValue N0,
SDValue N1,
bool IsForward) {
13651 LoadSDNode *Load0 = dyn_cast<LoadSDNode>(N0);
13652 LoadSDNode *Load1 = dyn_cast<LoadSDNode>(N1);
13661 if (!BaseLocDecomp0.getBase() ||
13662 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13663 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13665 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13667 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13677 if (IsBefore < 0) {
13680 }
else if (IsBefore > 0) {
13693 }
else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13703 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13713 if (
SDValue R = DistrubuteVecReduceLoad(N0, N1,
true))
13715 if (
SDValue R = DistrubuteVecReduceLoad(N1, N0,
false))
13722 if (!Subtarget->hasMVEIntegerOps())
13728 EVT VT =
N->getValueType(0);
13733 if (VT != MVT::i64)
13744 auto MakeVecReduce = [&](
unsigned Opcode,
unsigned OpcodeA,
SDValue NA,
13762 std::tie(Ops[0], Ops[1]) = DAG.
SplitScalar(NA, dl, MVT::i32, MVT::i32);
13764 unsigned S = VecRed->
getOpcode() == OpcodeA ? 2 : 0;
13813 "Expected shift op");
13831 if (
auto *Const = dyn_cast<ConstantSDNode>(N1->
getOperand(1))) {
13832 if (Const->getAPIntValue().ult(256))
13835 Const->getAPIntValue().sgt(-256))
13851 (
N->getOperand(0).getOpcode() ==
ISD::SHL ||
13852 N->getOperand(0).getOpcode() ==
ISD::SRL) &&
13853 "Expected XOR(SHIFT) pattern");
13856 auto *XorC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13857 auto *ShiftC = dyn_cast<ConstantSDNode>(
N->getOperand(0).getOperand(1));
13858 if (XorC && ShiftC) {
13859 unsigned MaskIdx, MaskLen;
13860 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13861 unsigned ShiftAmt = ShiftC->getZExtValue();
13862 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
13863 if (
N->getOperand(0).getOpcode() ==
ISD::SHL)
13864 return MaskIdx == ShiftAmt && MaskLen == (
BitWidth - ShiftAmt);
13865 return MaskIdx == 0 && MaskLen == (
BitWidth - ShiftAmt);
13875 N->getOperand(0).getOpcode() ==
ISD::SRL) ||
13877 N->getOperand(0).getOpcode() ==
ISD::SHL)) &&
13878 "Expected shift-shift mask");
13891 return Subtarget->hasMVEIntegerOps() &&
isTypeLegal(VT);
13895 if (!Subtarget->hasNEON()) {
13914 return Subtarget->hasFP64();
13917 return Subtarget->hasMVEFloatOps();
13946 if (ST->isThumb() && ST->isThumb1Only())
13950 for (
auto *U :
N->uses()) {
13951 switch(U->getOpcode()) {
13964 if (isa<ConstantSDNode>(U->getOperand(0)) ||
13965 isa<ConstantSDNode>(U->getOperand(1)))
13969 if (U->getOperand(0).getOpcode() ==
ISD::SHL ||
13970 U->getOperand(1).getOpcode() ==
ISD::SHL)
13980 if (
N->getOperand(0).getOpcode() !=
ISD::SHL)
13985 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13986 auto *C2 = dyn_cast<ConstantSDNode>(SHL.getOperand(1));
13987 if (!C1ShlC2 || !C2)
13990 APInt C2Int = C2->getAPIntValue();
13991 APInt C1Int = C1ShlC2->getAPIntValue();
13993 if (C2Int.
uge(C2Width))
13999 if ((C1Int & Mask) != C1Int)
14006 auto LargeImm = [](
const APInt &Imm) {
14007 unsigned Zeros = Imm.countl_zero() + Imm.countr_zero();
14008 return Imm.getBitWidth() - Zeros > 8;
14011 if (LargeImm(C1Int) || LargeImm(C2Int))
14023 SHL.dump();
N->dump());
14058 if (CSINC.getOpcode() !=
ARMISD::CSINC || !CSINC.hasOneUse())
14067 CSINC.getOperand(0)),
14068 CSINC.getOperand(1), CSINC.getOperand(2),
14069 CSINC.getOperand(3));
14088 if (!Subtarget->hasMVEIntegerOps() || !
N->getValueType(0).isVector())
14130 if (!Subtarget->hasVMLxForwarding())
14149 EVT VT =
N->getValueType(0);
14160 EVT VT =
N->getValueType(0);
14161 if (VT != MVT::v2i64)
14170 EVT VT = cast<VTSDNode>(
Op->getOperand(1))->getVT();
14172 return Op->getOperand(0);
14186 And =
And->getOperand(0);
14191 Mask = Mask->getOperand(0);
14194 Mask.getValueType() != MVT::v4i32)
14200 return And->getOperand(0);
14205 if (
SDValue Op0 = IsSignExt(N0)) {
14206 if (
SDValue Op1 = IsSignExt(N1)) {
14212 if (
SDValue Op0 = IsZeroExt(N0)) {
14213 if (
SDValue Op1 = IsZeroExt(N1)) {
14228 EVT VT =
N->getValueType(0);
14229 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14240 if (VT != MVT::i32)
14247 int64_t MulAmt =
C->getSExtValue();
14248 unsigned ShiftAmt = llvm::countr_zero<uint64_t>(MulAmt);
14250 ShiftAmt = ShiftAmt & (32 - 1);
14255 MulAmt >>= ShiftAmt;
14258 if (llvm::has_single_bit<uint32_t>(MulAmt - 1)) {
14266 }
else if (llvm::has_single_bit<uint32_t>(MulAmt + 1)) {
14278 if (llvm::has_single_bit<uint32_t>(MulAmtAbs + 1)) {
14286 }
else if (llvm::has_single_bit<uint32_t>(MulAmtAbs - 1)) {
14316 if (
N->getValueType(0) != MVT::i32)
14325 if (C1 == 255 || C1 == 65535)
14328 SDNode *N0 =
N->getOperand(0).getNode();
14342 if (!C2 || C2 >= 32)
14386 if (Trailing == C2 && C2 + C3 < 32) {
14399 if (Leading == C2 && C2 + C3 < 32) {
14427 EVT VT =
N->getValueType(0);
14431 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14434 APInt SplatBits, SplatUndef;
14435 unsigned SplatBitSize;
14437 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14438 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14439 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14440 SplatBitSize == 64) {
14474 if (!Subtarget->hasV6Ops() ||
14475 (Subtarget->isThumb() &&
14476 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14479 SDValue SRL = OR->getOperand(0);
14480 SDValue SHL = OR->getOperand(1);
14483 SRL = OR->getOperand(1);
14484 SHL = OR->getOperand(0);
14491 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) ||
14495 SDNode *SMULLOHI = SRL.getOperand(0).getNode();
14496 if (SRL.getOperand(0) !=
SDValue(SMULLOHI, 0) ||
14497 SHL.getOperand(0) !=
SDValue(SMULLOHI, 1))
14516 unsigned Opcode = 0;
14517 if (
isS16(OpS16, DAG))
14535 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6T2Ops())
14538 EVT VT =
N->getValueType(0);
14553 if (VT != MVT::i32)
14566 if (Mask == 0xffff)
14573 if ((Val & ~Mask) != Val)
14598 (Mask == ~Mask2)) {
14601 if (Subtarget->hasDSP() &&
14602 (Mask == 0xffff || Mask == 0xffff0000))
14615 (~Mask == Mask2)) {
14618 if (Subtarget->hasDSP() &&
14619 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14692 EVT VT =
N->getValueType(0);
14697 auto IsFreelyInvertable = [&](
SDValue V) {
14704 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14720 EVT VT =
N->getValueType(0);
14726 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14727 VT == MVT::v8i1 || VT == MVT::v16i1))
14730 APInt SplatBits, SplatUndef;
14731 unsigned SplatBitSize;
14733 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14734 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14735 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14736 SplatBitSize == 64) {
14772 unsigned SplatBitSize;
14775 APInt SplatBits0, SplatBits1;
14779 if (BVN0 && BVN0->
isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
14780 HasAnyUndefs) && !HasAnyUndefs) {
14781 if (BVN1 && BVN1->
isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
14782 HasAnyUndefs) && !HasAnyUndefs) {
14787 SplatBits0 == ~SplatBits1) {
14817 EVT VT =
N->getValueType(0);
14832 if (Subtarget->hasMVEIntegerOps()) {
14863 ToMask = ~N->getConstantOperandAPInt(2);
14869 isa<ConstantSDNode>(
From->getOperand(1))) {
14870 APInt Shift =
From->getConstantOperandAPInt(1);
14883 unsigned LastActiveBitInA =
A.countr_zero();
14884 unsigned FirstActiveBitInB =
B.getBitWidth() -
B.countl_zero() - 1;
14885 return LastActiveBitInA - 1 == FirstActiveBitInB;
14890 APInt ToMask, FromMask;
14898 APInt NewToMask, NewFromMask;
14900 if (NewFrom !=
From)
14904 if ((NewToMask & ToMask).getBoolValue())
14929 unsigned InvMask =
N->getConstantOperandVal(2);
14931 unsigned Width = llvm::bit_width<unsigned>(~InvMask) - LSB;
14933 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14934 "undefined behavior");
14935 unsigned Mask = (1u << Width) - 1;
14937 if ((Mask & (~Mask2)) == 0)
14939 N->getOperand(0), N1.
getOperand(0),
N->getOperand(2));
14946 APInt ToMask1, FromMask1;
14949 APInt ToMask2, FromMask2;
14955 APInt NewFromMask = FromMask1 | FromMask2;
14956 APInt NewToMask = ToMask1 | ToMask2;
14958 EVT VT =
N->getValueType(0);
14961 if (NewFromMask[0] == 0)
14973 APInt ToMask1 = ~N->getConstantOperandAPInt(2);
14974 APInt ToMask2 = ~N0.getConstantOperandAPInt(2);
14976 if (!N0.
hasOneUse() || (ToMask1 & ToMask2) != 0 ||
14980 EVT VT =
N->getValueType(0);
14983 N->getOperand(1),
N->getOperand(2));
14997 SDValue CSInc = Cmp->getOperand(0);
15047 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ)
15048 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15051 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
15053 N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15066 SDValue InDouble =
N->getOperand(0);
15075 !cast<LoadSDNode>(InNode)->isVolatile()) {
15081 SDValue BasePtr = LD->getBasePtr();
15083 DAG.
getLoad(MVT::i32,
DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
15084 LD->getAlign(), LD->getMemOperand()->getFlags());
15090 LD->getPointerInfo().getWithOffset(4),
15092 LD->getMemOperand()->getFlags());
15104 isa<ConstantSDNode>(InDouble.
getOperand(1))) {
15125 if (!Subtarget->
isLittle() && BVSwap)
15135 if (isa<ConstantSDNode>(BV.
getOperand(2))) {
15143 if (!Subtarget->
isLittle() && BVSwap)
15188 if (Copy.getValueType() == MVT::f32 &&
15190 bool HasGlue = Copy->getNumOperands() == 3;
15191 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1),
15192 HasGlue ? Copy->getOperand(2) :
SDValue()};
15193 EVT OutTys[] = {
N->getValueType(0), MVT::Other, MVT::Glue};
15211 if (
LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Op0)) {
15212 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15213 LN0->getMemoryVT() == MVT::i16) {
15216 LN0->getBasePtr(), LN0->getMemOperand());
15234 EVT VT =
N->getValueType(0);
15268 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
15269 for (
unsigned i = 0; i < NumElts; ++i) {
15270 SDNode *Elt =
N->getOperand(i).getNode();
15287 if (
N->getNumOperands() == 2)
15293 EVT VT =
N->getValueType(0);
15299 for (
unsigned i = 0; i < NumElts; ++i) {
15325 EVT VT =
N->getValueType(0);
15333 assert(EltVT == MVT::f32 &&
"Unexpected type!");
15338 Use->getValueType(0).isFloatingPoint())
15346 unsigned NumOfBitCastedElts = 0;
15348 unsigned NumOfRelevantElts = NumElts;
15349 for (
unsigned Idx = 0;
Idx < NumElts; ++
Idx) {
15354 ++NumOfBitCastedElts;
15355 }
else if (Elt.
isUndef() || isa<ConstantSDNode>(Elt))
15358 --NumOfRelevantElts;
15362 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15380 for (
unsigned Idx = 0 ;
Idx < NumElts; ++
Idx) {
15385 V->getOperand(0).getValueType() == MVT::i32)
15387 V = V.getOperand(0);
15404 EVT VT =
N->getValueType(0);
15411 if (
Op->getOperand(0).getValueType() == VT)
15412 return Op->getOperand(0);
15427 if (
Op.getValueType() == MVT::i32) {
15438 EVT VT =
N->getValueType(0);
15443 if (ST->isLittle())
15453 if (
Op->getOperand(0).getValueType() == VT)
15454 return Op->getOperand(0);
15463 if (!Subtarget->hasMVEIntegerOps())
15466 EVT VT =
N->getValueType(0);
15497 EVT VT =
N->getValueType(0);
15498 SDNode *Elt =
N->getOperand(1).getNode();
15513 Vec, V,
N->getOperand(2));
15523 EVT VT =
N->getValueType(0);
15532 Ext.getOperand(0).getValueType() == MVT::f32)
15533 Ext = Ext.getOperand(0);
15535 !isa<ConstantSDNode>(Ext.getOperand(1)) ||
15536 Ext.getConstantOperandVal(1) % 2 != 0)
15538 if (Ext->use_size() == 1 &&
15543 SDValue Op0 = Ext.getOperand(0);
15546 unsigned Lane = Ext.getConstantOperandVal(1);
15552 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15553 isa<ConstantSDNode>(V->getOperand(1)) &&
15554 V->getConstantOperandVal(1) == Lane + 1 &&
15555 V->getOperand(0).getResNo() == ResNo;
15557 if (OtherIt == Op0->
uses().end())
15562 SDValue OtherExt(*OtherIt, 0);
15575 DCI.
DAG.
getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32));
15587 EVT VT =
N->getValueType(0);
15593 if (VT == MVT::f16 &&
X.getValueType() == MVT::i32)
15595 if (VT == MVT::i32 &&
X.getValueType() == MVT::f16)
15597 if (VT == MVT::f32 &&
X.getValueType() == MVT::i32)
15600 while (
X.getValueType() != VT &&
X->getOpcode() ==
ISD::BITCAST)
15601 X =
X->getOperand(0);
15602 if (
X.getValueType() == VT)
15608 isa<ConstantSDNode>(
N->getOperand(1)) &&
15610 return Op0.
getOperand(
N->getConstantOperandVal(1));
15615 isa<ConstantSDNode>(
N->getOperand(1)) &&
15620 unsigned Offset =
N->getConstantOperandVal(1);
15632 unsigned Idx =
N->getConstantOperandVal(1);
15646 EVT VT =
N->getValueType(0);
15650 cast<VTSDNode>(
N->getOperand(1))->getVT() ==
15651 Op.getOperand(0).getValueType().getScalarType())
15661 SDValue SubVec =
N->getOperand(1);
15662 uint64_t IdxVal =
N->getConstantOperandVal(2);
15673 if (IdxVal == 0 && Vec.
isUndef())
15679 (IdxVal != 0 && IdxVal != NumSubElts))
15752 EVT VT =
N->getValueType(0);
15763 unsigned HalfElts = NumElts/2;
15765 for (
unsigned n = 0; n < NumElts; ++n) {
15768 if (MaskElt < (
int)HalfElts)
15770 else if (MaskElt >= (
int)NumElts && MaskElt < (
int)(NumElts + HalfElts))
15771 NewElt = HalfElts + MaskElt - NumElts;
15798 bool SimpleConstIncOnly,
15806 bool isLoadOp =
true;
15807 bool isLaneOp =
false;
15810 bool hasAlignment =
true;
15811 unsigned NewOpc = 0;
15812 unsigned NumVecs = 0;
15813 if (
Target.isIntrinsic) {
15814 unsigned IntNo =
N->getConstantOperandVal(1);
15818 case Intrinsic::arm_neon_vld1:
15822 case Intrinsic::arm_neon_vld2:
15826 case Intrinsic::arm_neon_vld3:
15830 case Intrinsic::arm_neon_vld4:
15834 case Intrinsic::arm_neon_vld1x2:
15837 hasAlignment =
false;
15839 case Intrinsic::arm_neon_vld1x3:
15842 hasAlignment =
false;
15844 case Intrinsic::arm_neon_vld1x4:
15847 hasAlignment =
false;
15849 case Intrinsic::arm_neon_vld2dup:
15853 case Intrinsic::arm_neon_vld3dup:
15857 case Intrinsic::arm_neon_vld4dup:
15861 case Intrinsic::arm_neon_vld2lane:
15866 case Intrinsic::arm_neon_vld3lane:
15871 case Intrinsic::arm_neon_vld4lane:
15876 case Intrinsic::arm_neon_vst1:
15881 case Intrinsic::arm_neon_vst2:
15886 case Intrinsic::arm_neon_vst3:
15891 case Intrinsic::arm_neon_vst4:
15896 case Intrinsic::arm_neon_vst2lane:
15902 case Intrinsic::arm_neon_vst3lane:
15908 case Intrinsic::arm_neon_vst4lane:
15914 case Intrinsic::arm_neon_vst1x2:
15918 hasAlignment =
false;
15920 case Intrinsic::arm_neon_vst1x3:
15924 hasAlignment =
false;
15926 case Intrinsic::arm_neon_vst1x4:
15930 hasAlignment =
false;
15935 switch (
N->getOpcode()) {
15971 VecTy =
N->getValueType(0);
15972 }
else if (
Target.isIntrinsic) {
15973 VecTy =
N->getOperand(
Target.AddrOpIdx + 1).getValueType();
15976 "Node has to be a load, a store, or an intrinsic!");
15977 VecTy =
N->getOperand(1).getValueType();
15985 if (isLaneOp || isVLDDUPOp)
15988 if (NumBytes >= 3 * 16 &&
User.ConstInc != NumBytes) {
15994 if (SimpleConstIncOnly &&
User.ConstInc != NumBytes)
16000 EVT AlignedVecTy = VecTy;
16017 if (isa<LSBaseSDNode>(
N)) {
16020 assert(NumVecs == 1 &&
"Unexpected multi-element generic load/store.");
16021 assert(!isLaneOp &&
"Unexpected generic load/store lane.");
16032 Alignment =
Align(1);
16038 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16040 for (n = 0; n < NumResultVecs; ++n)
16041 Tys[n] = AlignedVecTy;
16042 Tys[n++] = MVT::i32;
16043 Tys[n] = MVT::Other;
16058 unsigned LastOperand =
16059 hasAlignment ?
N->getNumOperands() - 1 :
N->getNumOperands();
16060 for (
unsigned i =
Target.AddrOpIdx + 1; i < LastOperand; ++i)
16069 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::STORE) {
16080 for (
unsigned i = 0; i < NumResultVecs; ++i)
16085 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::LOAD) {
16086 SDValue &LdVal = NewResults[0];
16122 switch (
N->getOpcode()) {
16125 if (isa<ConstantSDNode>(
N->getOperand(1))) {
16126 *
Ptr =
N->getOperand(0);
16127 *CInc =
N->getOperand(1);
16133 if (isa<ConstantSDNode>(
N->getOperand(2))) {
16134 *
Ptr =
N->getOperand(1);
16135 *CInc =
N->getOperand(2);
16170 const unsigned AddrOpIdx = ((isIntrinsic ||
isStore) ? 2 : 1);
16179 UE =
Addr.getNode()->use_end(); UI != UE; ++UI) {
16181 if (UI.getUse().getResNo() !=
Addr.getResNo() ||
16186 unsigned ConstInc =
16204 if (UI.getUse().getResNo() !=
Base.getResNo() ||
User ==
Addr.getNode() ||
16209 unsigned UserOffset =
16212 if (!UserOffset || UserOffset <=
Offset)
16215 unsigned NewConstInc = UserOffset -
Offset;
16225 unsigned NumValidUpd = BaseUpdates.
size();
16226 for (
unsigned I = 0;
I < NumValidUpd;) {
16230 std::swap(BaseUpdates[
I], BaseUpdates[NumValidUpd]);
16238 BaseUpdates.
resize(NumValidUpd);
16243 std::stable_sort(BaseUpdates.
begin(), BaseUpdates.
end(),
16245 return LHS.ConstInc < RHS.ConstInc;
16274 unsigned IntNo =
N->getConstantOperandVal(1);
16275 if (IntNo == Intrinsic::arm_mve_vst2q &&
N->getConstantOperandVal(5) != 1)
16277 if (IntNo == Intrinsic::arm_mve_vst4q &&
N->getConstantOperandVal(7) != 3)
16282 UE =
Addr.getNode()->use_end();
16286 UI.getUse().getResNo() !=
Addr.getResNo())
16302 bool isLoadOp =
true;
16303 unsigned NewOpc = 0;
16304 unsigned NumVecs = 0;
16308 case Intrinsic::arm_mve_vld2q:
16312 case Intrinsic::arm_mve_vld4q:
16316 case Intrinsic::arm_mve_vst2q:
16321 case Intrinsic::arm_mve_vst4q:
16331 VecTy =
N->getValueType(0);
16333 VecTy =
N->getOperand(3).getValueType();
16347 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16349 for (n = 0; n < NumResultVecs; ++n)
16351 Tys[n++] = MVT::i32;
16352 Tys[n] = MVT::Other;
16361 for (
unsigned i = 3; i <
N->getNumOperands(); ++i)
16369 for (
unsigned i = 0; i < NumResultVecs; ++i)
16388 EVT VT =
N->getValueType(0);
16394 SDNode *VLD =
N->getOperand(0).getNode();
16397 unsigned NumVecs = 0;
16398 unsigned NewOpc = 0;
16400 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16403 }
else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16406 }
else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16419 if (UI.getUse().getResNo() == NumVecs)
16423 VLDLaneNo !=
User->getConstantOperandVal(1))
16430 for (n = 0; n < NumVecs; ++n)
16432 Tys[n] = MVT::Other;
16443 unsigned ResNo = UI.getUse().
getResNo();
16445 if (ResNo == NumVecs)
16453 std::vector<SDValue> VLDDupResults;
16454 for (
unsigned n = 0; n < NumVecs; ++n)
16468 EVT VT =
N->getValueType(0);
16471 if (Subtarget->hasMVEIntegerOps()) {
16475 ExtractVT = MVT::i32;
16477 N->getOperand(0),
N->getOperand(1));
16489 Op =
Op.getOperand(0);
16494 unsigned EltSize =
Op.getScalarValueSizeInBits();
16496 unsigned Imm =
Op.getConstantOperandVal(0);
16512 if (Subtarget->hasMVEIntegerOps()) {
16515 if (
Op.getValueType() == MVT::f32)
16518 else if (
Op.getValueType() == MVT::f16)
16523 if (!Subtarget->hasNEON())
16530 if (LD &&
Op.hasOneUse() && LD->isUnindexed() &&
16531 LD->getMemoryVT() ==
N->getValueType(0).getVectorElementType()) {
16532 SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1),
16537 LD->getMemoryVT(), LD->getMemOperand());
16548 EVT VT =
N->getValueType(0);
16570 assert(StVT != VT &&
"Cannot truncate to the same type");
16580 if (0 != (NumElems * FromEltSz) % ToEltSz)
16583 unsigned SizeRatio = FromEltSz / ToEltSz;
16588 NumElems * SizeRatio);
16594 for (
unsigned i = 0; i < NumElems; ++i)
16608 MVT StoreType = MVT::i8;
16610 if (TLI.
isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16629 unsigned E = (ToEltSz * NumElems) / StoreType.
getSizeInBits();
16630 for (
unsigned I = 0;
I < E;
I++) {
16661 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16664 unsigned NumElements = 4;
16681 unsigned Off0 = Rev ? NumElts : 0;
16682 unsigned Off1 = Rev ? 0 : NumElts;
16684 for (
unsigned I = 0;
I < NumElts;
I += 2) {
16685 if (M[
I] >= 0 && M[
I] != (
int)(Off0 +
I / 2))
16687 if (M[
I + 1] >= 0 && M[
I + 1] != (
int)(Off1 +
I / 2))
16694 if (
auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Trunc.
getOperand(0)))
16695 if (isVMOVNShuffle(Shuffle,
false) || isVMOVNShuffle(Shuffle,
true))
16715 unsigned NewOffset = i * NumElements * ToEltVT.
getSizeInBits() / 8;
16730 NewToVT, Alignment, MMOFlags, AAInfo);
16763 unsigned NewOffset =
16771 NewToVT, Alignment, MMOFlags, AAInfo);
16793 {Extract.getOperand(0), Extract.getOperand(1)});
16824 if (Subtarget->hasNEON())
16828 if (Subtarget->hasMVEFloatOps())
16832 if (Subtarget->hasMVEIntegerOps()) {
16908 if (!Subtarget->hasNEON())
16912 if (!
Op.getValueType().isVector() || !
Op.getValueType().isSimple() ||
16917 if (!isa<BuildVectorSDNode>(ConstVec))
16920 MVT FloatTy =
Op.getSimpleValueType().getVectorElementType();
16922 MVT IntTy =
N->getSimpleValueType(0).getVectorElementType();
16924 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
16925 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
16936 if (
C == -1 ||
C == 0 ||
C > 32)
16941 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
16942 Intrinsic::arm_neon_vcvtfp2fxu;
16945 DAG.
getConstant(IntrinsicOpcode, dl, MVT::i32),
Op->getOperand(0),
16948 if (IntBits < FloatBits)
16956 if (!Subtarget->hasMVEFloatOps())
16964 EVT VT =
N->getValueType(0);
16969 auto isIdentitySplat = [&](
SDValue Op,
bool NSZ) {
16973 uint64_t ImmVal =
Op.getOperand(0).getConstantOperandVal(0);
16974 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
16976 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
16989 if (!isIdentitySplat(Op1.
getOperand(2), NSZ))
17000 EVT VT =
N->getValueType(0);
17003 if (!
N->getFlags().hasAllowReassociation())
17010 unsigned Opc =
A.getConstantOperandVal(0);
17011 if (Opc != Intrinsic::arm_mve_vcmlaq)
17016 A.getOperand(3),
A.getOperand(4));
17048 if (!Subtarget->hasNEON())
17052 unsigned OpOpcode =
Op.getNode()->getOpcode();
17053 if (!
N->getValueType(0).isVector() || !
N->getValueType(0).isSimple() ||
17057 SDValue ConstVec =
N->getOperand(1);
17058 if (!isa<BuildVectorSDNode>(ConstVec))
17061 MVT FloatTy =
N->getSimpleValueType(0).getVectorElementType();
17063 MVT IntTy =
Op.getOperand(0).getSimpleValueType().getVectorElementType();
17065 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
17066 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17086 int32_t
C = IntVal.exactLogBase2();
17087 if (
C == -1 ||
C == 0 ||
C > 32)
17093 if (IntBits < FloatBits)
17095 NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, ConvInput);
17097 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfxs2fp
17098 : Intrinsic::arm_neon_vcvtfxu2fp;
17106 if (!ST->hasMVEIntegerOps())
17110 EVT ResVT =
N->getValueType(0);
17138 EVT AVT =
A.getValueType();
17144 auto ExtendIfNeeded = [&](
SDValue A,
unsigned ExtendCode) {
17145 EVT AVT =
A.getValueType();
17157 if (ExtTypeMatches(
A, ExtTypes))
17158 return ExtendIfNeeded(
A, ExtendCode);
17161 auto IsPredVADDV = [&](
MVT RetTy,
unsigned ExtendCode,
17168 if (Ext->getOpcode() != ExtendCode)
17171 if (ExtTypeMatches(
A, ExtTypes))
17172 return ExtendIfNeeded(
A, ExtendCode);
17185 if (ResVT !=
RetTy)
17188 if (
Mul->getOpcode() == ExtendCode &&
17189 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17191 Mul =
Mul->getOperand(0);
17200 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17201 A = ExtendIfNeeded(
A, ExtendCode);
17202 B = ExtendIfNeeded(
B, ExtendCode);
17220 if (
Mul->getOpcode() == ExtendCode &&
17221 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17223 Mul =
Mul->getOperand(0);
17232 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17233 A = ExtendIfNeeded(
A, ExtendCode);
17234 B = ExtendIfNeeded(
B, ExtendCode);
17245 EVT VT = Ops[0].getValueType();
17246 if (VT == MVT::v16i8) {
17248 "Unexpected illegal long reduction opcode");
17253 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
17256 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
17277 if (IsVMLAV(MVT::i64,
ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17280 if (IsVMLAV(MVT::i64,
ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17344 Op =
Op->getOperand(1);
17346 Op->getOperand(0)->getOpcode() ==
ISD::MUL) {
17348 if (
Mul->getOperand(0) ==
Mul->getOperand(1) &&
17365 unsigned VecOp =
N->getOperand(0).getValueType().isVector() ? 0 : 2;
17366 auto *Shuf = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp));
17367 if (!Shuf || !Shuf->getOperand(1).isUndef())
17372 APInt SetElts(Mask.size(), 0);
17373 for (
int E : Mask) {
17374 if (E < 0 || E >= (
int)Mask.size())
17381 if (
N->getNumOperands() != VecOp + 1) {
17382 auto *Shuf2 = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp + 1));
17383 if (!Shuf2 || !Shuf2->getOperand(1).isUndef() || Shuf2->getMask() != Mask)
17389 if (
Op.getValueType().isVector())
17401 unsigned IsTop =
N->getConstantOperandVal(2);
17408 if (Op0->
isUndef() && !IsTop)
17422 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17424 APInt Op0DemandedElts =
17425 IsTop ? Op1DemandedElts
17440 unsigned IsTop =
N->getConstantOperandVal(2);
17442 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17443 APInt Op0DemandedElts =
17455 EVT VT =
N->getValueType(0);
17459 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(
LHS);
17460 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(
RHS);
17462 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
17463 LHS.getOperand(1).isUndef() &&
RHS.getOperand(1).isUndef() &&
17467 LHS.getOperand(0),
RHS.getOperand(0));
17481 if (
auto C = dyn_cast<ConstantSDNode>(
N->getOperand(2))) {
17482 int ShiftAmt =
C->getSExtValue();
17483 if (ShiftAmt == 0) {
17489 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17490 unsigned NewOpcode =
17506 unsigned IntNo =
N->getConstantOperandVal(0);
17517 case Intrinsic::arm_neon_vshifts:
17518 case Intrinsic::arm_neon_vshiftu:
17519 case Intrinsic::arm_neon_vrshifts:
17520 case Intrinsic::arm_neon_vrshiftu:
17521 case Intrinsic::arm_neon_vrshiftn:
17522 case Intrinsic::arm_neon_vqshifts:
17523 case Intrinsic::arm_neon_vqshiftu:
17524 case Intrinsic::arm_neon_vqshiftsu:
17525 case Intrinsic::arm_neon_vqshiftns:
17526 case Intrinsic::arm_neon_vqshiftnu:
17527 case Intrinsic::arm_neon_vqshiftnsu:
17528 case Intrinsic::arm_neon_vqrshiftns:
17529 case Intrinsic::arm_neon_vqrshiftnu:
17530 case Intrinsic::arm_neon_vqrshiftnsu: {
17531 EVT VT =
N->getOperand(1).getValueType();
17533 unsigned VShiftOpc = 0;
17536 case Intrinsic::arm_neon_vshifts:
17537 case Intrinsic::arm_neon_vshiftu:
17542 if (
isVShiftRImm(
N->getOperand(2), VT,
false,
true, Cnt)) {
17549 case Intrinsic::arm_neon_vrshifts:
17550 case Intrinsic::arm_neon_vrshiftu:
17555 case Intrinsic::arm_neon_vqshifts:
17556 case Intrinsic::arm_neon_vqshiftu:
17561 case Intrinsic::arm_neon_vqshiftsu:
17566 case Intrinsic::arm_neon_vrshiftn:
17567 case Intrinsic::arm_neon_vqshiftns:
17568 case Intrinsic::arm_neon_vqshiftnu:
17569 case Intrinsic::arm_neon_vqshiftnsu:
17570 case Intrinsic::arm_neon_vqrshiftns:
17571 case Intrinsic::arm_neon_vqrshiftnu:
17572 case Intrinsic::arm_neon_vqrshiftnsu:
17584 case Intrinsic::arm_neon_vshifts:
17585 case Intrinsic::arm_neon_vshiftu:
17588 case Intrinsic::arm_neon_vrshifts:
17591 case Intrinsic::arm_neon_vrshiftu:
17594 case Intrinsic::arm_neon_vrshiftn:
17597 case Intrinsic::arm_neon_vqshifts:
17600 case Intrinsic::arm_neon_vqshiftu:
17603 case Intrinsic::arm_neon_vqshiftsu:
17606 case Intrinsic::arm_neon_vqshiftns:
17609 case Intrinsic::arm_neon_vqshiftnu:
17612 case Intrinsic::arm_neon_vqshiftnsu:
17615 case Intrinsic::arm_neon_vqrshiftns:
17618 case Intrinsic::arm_neon_vqrshiftnu:
17621 case Intrinsic::arm_neon_vqrshiftnsu:
17627 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17628 N->getOperand(1), DAG.
getConstant(Cnt, dl, MVT::i32));
17631 case Intrinsic::arm_neon_vshiftins: {
17632 EVT VT =
N->getOperand(1).getValueType();
17634 unsigned VShiftOpc = 0;
17638 else if (
isVShiftRImm(
N->getOperand(3), VT,
false,
true, Cnt))
17645 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17646 N->getOperand(1),
N->getOperand(2),
17650 case Intrinsic::arm_neon_vqrshifts:
17651 case Intrinsic::arm_neon_vqrshiftu:
17655 case Intrinsic::arm_mve_vqdmlah:
17656 case Intrinsic::arm_mve_vqdmlash:
17657 case Intrinsic::arm_mve_vqrdmlah:
17658 case Intrinsic::arm_mve_vqrdmlash:
17659 case Intrinsic::arm_mve_vmla_n_predicated:
17660 case Intrinsic::arm_mve_vmlas_n_predicated:
17661 case Intrinsic::arm_mve_vqdmlah_predicated:
17662 case Intrinsic::arm_mve_vqdmlash_predicated:
17663 case Intrinsic::arm_mve_vqrdmlah_predicated:
17664 case Intrinsic::arm_mve_vqrdmlash_predicated: {
17669 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
17676 case Intrinsic::arm_mve_minv:
17677 case Intrinsic::arm_mve_maxv:
17678 case Intrinsic::arm_mve_minav:
17679 case Intrinsic::arm_mve_maxav:
17680 case Intrinsic::arm_mve_minv_predicated:
17681 case Intrinsic::arm_mve_maxv_predicated:
17682 case Intrinsic::arm_mve_minav_predicated:
17683 case Intrinsic::arm_mve_maxav_predicated: {
17686 unsigned BitWidth =
N->getOperand(2)->getValueType(0).getScalarSizeInBits();
17693 case Intrinsic::arm_mve_addv: {
17696 bool Unsigned =
N->getConstantOperandVal(2);
17701 case Intrinsic::arm_mve_addlv:
17702 case Intrinsic::arm_mve_addlv_predicated: {
17705 bool Unsigned =
N->getConstantOperandVal(2);
17706 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
17711 for (
unsigned i = 1, e =
N->getNumOperands(); i < e; i++)
17734 EVT VT =
N->getValueType(0);
17736 if (ST->isThumb1Only() &&
N->getOpcode() ==
ISD::SHL && VT == MVT::i32 &&
17737 N->getOperand(0)->getOpcode() ==
ISD::AND &&
17738 N->getOperand(0)->hasOneUse()) {
17746 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17755 if (AndMask == 255 || AndMask == 65535)
17759 if (MaskedBits > ShiftAmt) {
17774 if (ST->hasMVEIntegerOps())
17779 switch (
N->getOpcode()) {
17792 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
17793 unsigned VShiftOpc =
17796 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
17812 if (!LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed() ||
17815 EVT FromVT = LD->getValueType(0);
17816 EVT ToVT =
N->getValueType(0);
17823 unsigned NumElements = 0;
17824 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
17826 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
17828 if (NumElements == 0 ||
17838 SDValue BasePtr = LD->getBasePtr();
17839 Align Alignment = LD->getOriginalAlign();
17860 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
17861 Alignment, MMOFlags, AAInfo);
17867 if (FromEltVT == MVT::f16) {
17870 for (
unsigned i = 0; i < Loads.
size(); i++) {
17896 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
17900 EVT VT =
N->getValueType(0);
17904 if (VT == MVT::i32 &&
17905 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
17907 isa<ConstantSDNode>(Lane)) {
17910 switch (
N->getOpcode()) {
17924 if (ST->hasMVEIntegerOps())
17933 if (ST->hasMVEFloatOps())
17944 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
17948 EVT VT =
Op.getValueType();
17951 if (VT != MVT::i32 ||
17953 !isa<ConstantSDNode>(
Op.getOperand(1)) ||
17964 APInt MaxC = Max.getConstantOperandAPInt(1);
17967 !(MinC + 1).isPowerOf2())
17985 EVT VT =
N->getValueType(0);
17988 if (VT == MVT::i32)
17991 if (!ST->hasMVEIntegerOps())
17997 if (VT != MVT::v4i32 && VT != MVT::v8i16)
18000 auto IsSignedSaturate = [&](
SDNode *Min,
SDNode *Max) {
18008 if (VT == MVT::v4i32)
18009 SaturateC =
APInt(32, (1 << 15) - 1,
true);
18011 SaturateC =
APInt(16, (1 << 7) - 1,
true);
18018 MaxC != ~SaturateC)
18023 if (IsSignedSaturate(
N, N0.
getNode())) {
18026 if (VT == MVT::v4i32) {
18027 HalfVT = MVT::v8i16;
18028 ExtVT = MVT::v4i16;
18030 HalfVT = MVT::v16i8;
18045 auto IsUnsignedSaturate = [&](
SDNode *Min) {
18051 if (VT == MVT::v4i32)
18052 SaturateC =
APInt(32, (1 << 16) - 1,
true);
18054 SaturateC =
APInt(16, (1 << 8) - 1,
true);
18063 if (IsUnsignedSaturate(
N)) {
18067 if (VT == MVT::v4i32) {
18068 HalfVT = MVT::v8i16;
18069 ExtConst = 0x0000FFFF;
18071 HalfVT = MVT::v16i8;
18093 const APInt *CV = &
C->getAPIntValue();
18111 SDValue Op0 = CMOV->getOperand(0);
18112 SDValue Op1 = CMOV->getOperand(1);
18113 auto CC = CMOV->getConstantOperandAPInt(2).getLimitedValue();
18114 SDValue CmpZ = CMOV->getOperand(4);
18150 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18157 if ((OrCI & Known.
Zero) != OrCI)
18163 EVT VT =
X.getValueType();
18164 unsigned BitInX = AndC->
logBase2();
18172 for (
unsigned BitInY = 0, NumActiveBits = OrCI.
getActiveBits();
18173 BitInY < NumActiveBits; ++BitInY) {
18174 if (OrCI[BitInY] == 0)
18177 Mask.setBit(BitInY);
18194 switch (
N->getOpcode()) {
18198 if (!isa<ConstantSDNode>(
N.getOperand(1)))
18200 if (!cast<ConstantSDNode>(
N.getOperand(1))->isOne())
18206 auto *Const = dyn_cast<ConstantSDNode>(
N.getOperand(1));
18209 if (Const->isZero())
18211 else if (Const->isOne())
18215 CC = cast<CondCodeSDNode>(
N.getOperand(2))->get();
18219 unsigned IntOp =
N.getConstantOperandVal(1);
18220 if (IntOp != Intrinsic::test_start_loop_iterations &&
18221 IntOp != Intrinsic::loop_decrement_reg)
18247 bool Negate =
false;
18253 Cond =
N->getOperand(1);
18254 Dest =
N->getOperand(2);
18257 CC = cast<CondCodeSDNode>(
N->getOperand(1))->get();
18258 Cond =
N->getOperand(2);
18259 Dest =
N->getOperand(4);
18260 if (
auto *Const = dyn_cast<ConstantSDNode>(
N->getOperand(3))) {
18261 if (!Const->isOne() && !Const->isZero())
18263 Imm = Const->getZExtValue();
18291 assert((IsTrueIfZero(
CC, Imm) || IsFalseIfZero(
CC, Imm)) &&
18292 "unsupported condition");
18297 unsigned IntOp =
Int->getConstantOperandVal(1);
18299 &&
"expected single br user");
18300 SDNode *Br = *
N->use_begin();
18310 if (IntOp == Intrinsic::test_start_loop_iterations) {
18314 if (IsTrueIfZero(
CC, Imm)) {
18315 SDValue Ops[] = {Chain, Setup, Dest};
18320 UpdateUncondBr(Br, Dest, DAG);
18322 SDValue Ops[] = {Chain, Setup, OtherTarget};
18335 DAG.
getVTList(MVT::i32, MVT::Other), Args);
18343 if (
Target == OtherTarget)
18344 UpdateUncondBr(Br, Dest, DAG);
18363 EVT VT =
N->getValueType(0);
18382 LHS->getOperand(0)->getOperand(3),
LHS->getOperand(0)->getOperand(4));
18396 EVT VT =
N->getValueType(0);
18400 SDValue FalseVal =
N->getOperand(0);
18401 SDValue TrueVal =
N->getOperand(1);
18406 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops()) {
18432 N->getOperand(3), Cmp);
18437 N->getOperand(3), NewCmp);
18446 LHS->getOperand(2),
LHS->getOperand(3),
18447 LHS->getOperand(4));
18457 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ ||
18461 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
18463 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
18466 N->getOperand(3),
C);
18473 if (!Subtarget->
isThumb1Only() && Subtarget->hasV5TOps()) {
18509 N->getOperand(3), CPSRGlue.
getValue(1));
18524 N->getOperand(3), CPSRGlue.
getValue(1));
18542 const APInt *TrueConst;
18544 ((FalseVal.getOpcode() ==
ARMISD::SUBC && FalseVal.getOperand(0) ==
LHS &&
18545 FalseVal.getOperand(1) ==
RHS) ||
18549 unsigned ShiftAmount = TrueConst->
logBase2();
18564 if (Known.
Zero == 0xfffffffe)
18567 else if (Known.
Zero == 0xffffff00)
18570 else if (Known.
Zero == 0xffff0000)
18583 EVT DstVT =
N->getValueType(0);
18586 if (ST->hasMVEIntegerOps() && Src.getOpcode() ==
ARMISD::VDUP) {
18587 EVT SrcVT = Src.getValueType();
18595 Src = Src.getOperand(0);
18599 EVT SrcVT = Src.getValueType();
18619 EVT VT =
N->getValueType(0);
18627 if (
N->getNumOperands() == 2 &&
18631 N->getOperand(0).getOperand(1),
18632 N->getOperand(1).getOperand(0),
18633 N->getOperand(1).getOperand(1));
18636 if (
N->getNumOperands() == 2 &&
18639 auto *S0 = cast<ShuffleVectorSDNode>(
N->getOperand(0).getNode());
18640 auto *
S1 = cast<ShuffleVectorSDNode>(
N->getOperand(1).getNode());
18642 if (S0->getOperand(0) ==
S1->getOperand(0) &&
18643 S0->getOperand(1) ==
S1->getOperand(1)) {
18646 Mask.append(
S1->getMask().begin(),
S1->getMask().end());
18666 return Op.getOpcode() == ISD::BUILD_VECTOR ||
18667 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
18668 (Op.getOpcode() == ISD::BITCAST &&
18669 Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR);
18672 for (
unsigned Op = 0;
Op <
N->getNumOperands();
Op++) {
18674 for (
unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
18691 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18692 int NumIns =
N->getNumOperands();
18693 assert((NumIns == 2 || NumIns == 4) &&
18694 "Expected 2 or 4 inputs to an MVETrunc");
18696 if (
N->getNumOperands() == 4)
18700 for (
int I = 0;
I < NumIns;
I++) {
18702 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18722 if (!LD || !LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed())
18725 EVT FromVT = LD->getMemoryVT();
18726 EVT ToVT =
N->getValueType(0);
18733 unsigned NumElements = 0;
18734 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
18736 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
18738 assert(NumElements != 0);
18744 LD->getExtensionType() != NewExtType)
18751 SDValue BasePtr = LD->getBasePtr();
18752 Align Alignment = LD->getOriginalAlign();
18771 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
18772 Alignment, MMOFlags, AAInfo);
18788 EVT VT =
N->getValueType(0);
18790 assert(
N->getNumValues() == 2 &&
"Expected MVEEXT with 2 elements");
18791 assert((VT == MVT::v4i32 || VT == MVT::v8i16) &&
"Unexpected MVEEXT type");
18793 EVT ExtVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18795 auto Extend = [&](
SDValue V) {
18805 SDValue Ext = Extend(
N->getOperand(0));
18810 if (
auto *SVN = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(0))) {
18813 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
18818 auto CheckInregMask = [&](
int Start,
int Offset) {
18826 if (CheckInregMask(0, 0))
18828 else if (CheckInregMask(0, 1))
18829 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18830 else if (CheckInregMask(0, Mask.size()))
18832 else if (CheckInregMask(0, Mask.size() + 1))
18833 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18838 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18842 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18849 if (
N->getOperand(0)->getOpcode() ==
ISD::LOAD)
18859 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18860 int NumOuts =
N->getNumValues();
18861 assert((NumOuts == 2 || NumOuts == 4) &&
18862 "Expected 2 or 4 outputs to an MVEEXT");
18863 EVT LoadVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18865 if (
N->getNumOperands() == 4)
18871 StackPtr, MPI,
Align(4));
18874 for (
int I = 0;
I < NumOuts;
I++) {
18876 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18877 DAG.
getConstant(
I * 16 / NumOuts,
DL, StackPtr.getValueType()));
18882 VT, Chain,
Ptr, MPI, LoadVT,
Align(4));
18891 switch (
N->getOpcode()) {
19008 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19015 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19026 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19034 unsigned LowWidth =
N->getOperand(0).getValueType().getSizeInBits();
19036 unsigned HighWidth =
N->getOperand(1).getValueType().getSizeInBits();
19044 unsigned HighWidth =
N->getOperand(0).getValueType().getSizeInBits();
19046 unsigned LowWidth =
N->getOperand(1).getValueType().getSizeInBits();
19054 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19065 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19074 switch (
N->getConstantOperandVal(1)) {
19075 case Intrinsic::arm_neon_vld1:
19076 case Intrinsic::arm_neon_vld1x2:
19077 case Intrinsic::arm_neon_vld1x3:
19078 case Intrinsic::arm_neon_vld1x4:
19079 case Intrinsic::arm_neon_vld2:
19080 case Intrinsic::arm_neon_vld3:
19081 case Intrinsic::arm_neon_vld4:
19082 case Intrinsic::arm_neon_vld2lane:
19083 case Intrinsic::arm_neon_vld3lane:
19084 case Intrinsic::arm_neon_vld4lane:
19085 case Intrinsic::arm_neon_vld2dup:
19086 case Intrinsic::arm_neon_vld3dup:
19087 case Intrinsic::arm_neon_vld4dup:
19088 case Intrinsic::arm_neon_vst1:
19089 case Intrinsic::arm_neon_vst1x2:
19090 case Intrinsic::arm_neon_vst1x3:
19091 case Intrinsic::arm_neon_vst1x4:
19092 case Intrinsic::arm_neon_vst2:
19093 case Intrinsic::arm_neon_vst3:
19094 case Intrinsic::arm_neon_vst4:
19095 case Intrinsic::arm_neon_vst2lane:
19096 case Intrinsic::arm_neon_vst3lane:
19097 case Intrinsic::arm_neon_vst4lane:
19099 case Intrinsic::arm_mve_vld2q:
19100 case Intrinsic::arm_mve_vld4q:
19101 case Intrinsic::arm_mve_vst2q:
19102 case Intrinsic::arm_mve_vst4q:
19119 unsigned *
Fast)
const {
19128 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19130 if (AllowsUnaligned) {
19132 *
Fast = Subtarget->hasV7Ops();
19137 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19141 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->
isLittle())) {
19148 if (!Subtarget->hasMVEIntegerOps())
19152 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19153 Ty == MVT::v2i1)) {
19161 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19177 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19178 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19179 Ty == MVT::v2f64) {
19192 if ((
Op.isMemcpy() ||
Op.isZeroMemset()) && Subtarget->hasNEON() &&
19193 !FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat)) {
19195 if (
Op.size() >= 16 &&
19201 }
else if (
Op.size() >= 8 &&
19222 return (SrcBits == 64 && DestBits == 32);
19231 return (SrcBits == 64 && DestBits == 32);
19267 return Subtarget->hasFullFP16();
19277 return Ext->getType()->getScalarSizeInBits() ==
19278 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
19283 !areExtDoubled(cast<Instruction>(Ext1)) ||
19284 !areExtDoubled(cast<Instruction>(Ext2)))
19295 if (!
I->getType()->isVectorTy())
19298 if (Subtarget->hasNEON()) {
19299 switch (
I->getOpcode()) {
19300 case Instruction::Sub:
19301 case Instruction::Add: {
19313 if (!Subtarget->hasMVEIntegerOps())
19317 if (!
I->hasOneUse())
19319 auto *Sub = cast<Instruction>(*
I->users().begin());
19320 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(1) ==
I;
19330 switch (
I->getOpcode()) {
19331 case Instruction::Add:
19332 case Instruction::Mul:
19333 case Instruction::FAdd:
19334 case Instruction::ICmp:
19335 case Instruction::FCmp:
19337 case Instruction::FMul:
19338 return !IsFMSMul(
I);
19339 case Instruction::Sub:
19340 case Instruction::FSub:
19341 case Instruction::Shl:
19342 case Instruction::LShr:
19343 case Instruction::AShr:
19344 return Operand == 1;
19345 case Instruction::Call:
19346 if (
auto *
II = dyn_cast<IntrinsicInst>(
I)) {
19347 switch (
II->getIntrinsicID()) {
19348 case Intrinsic::fma:
19350 case Intrinsic::sadd_sat:
19351 case Intrinsic::uadd_sat:
19352 case Intrinsic::arm_mve_add_predicated:
19353 case Intrinsic::arm_mve_mul_predicated:
19354 case Intrinsic::arm_mve_qadd_predicated:
19355 case Intrinsic::arm_mve_vhadd:
19356 case Intrinsic::arm_mve_hadd_predicated:
19357 case Intrinsic::arm_mve_vqdmull:
19358 case Intrinsic::arm_mve_vqdmull_predicated:
19359 case Intrinsic::arm_mve_vqdmulh:
19360 case Intrinsic::arm_mve_qdmulh_predicated:
19361 case Intrinsic::arm_mve_vqrdmulh:
19362 case Intrinsic::arm_mve_qrdmulh_predicated:
19363 case Intrinsic::arm_mve_fma_predicated:
19365 case Intrinsic::ssub_sat:
19366 case Intrinsic::usub_sat:
19367 case Intrinsic::arm_mve_sub_predicated:
19368 case Intrinsic::arm_mve_qsub_predicated:
19369 case Intrinsic::arm_mve_hsub_predicated:
19370 case Intrinsic::arm_mve_vhsub:
19371 return Operand == 1;
19382 for (
auto OpIdx :
enumerate(
I->operands())) {
19383 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
19385 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
19389 if (Shuffle->
getOpcode() == Instruction::BitCast)
19390 Shuffle = dyn_cast<Instruction>(Shuffle->
getOperand(0));
19397 if (!IsSinker(
I, OpIdx.index()))
19402 for (
Use &U :
Op->uses()) {
19404 if (!IsSinker(
Insn, U.getOperandNo()))
19417 if (!Subtarget->hasMVEIntegerOps())
19435 if (
auto *Ld = dyn_cast<MaskedLoadSDNode>(ExtVal.
getOperand(0))) {
19436 if (Ld->isExpandingLoad())
19440 if (Subtarget->hasMVEIntegerOps())
19485bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(
const MachineFunction &MF,
19493 return Subtarget->hasMVEFloatOps();
19511 unsigned Scale = 1;
19528 if ((V & (Scale - 1)) != 0)
19530 return isUInt<5>(V / Scale);
19537 if (VT.
isVector() && Subtarget->hasNEON())
19540 !Subtarget->hasMVEFloatOps())
19543 bool IsNeg =
false;
19549 unsigned NumBytes = std::max((
unsigned)VT.
getSizeInBits() / 8, 1U);
19552 if (VT.
isVector() && Subtarget->hasMVEIntegerOps()) {
19556 return isShiftedUInt<7,2>(V);
19559 return isShiftedUInt<7,1>(V);
19561 return isUInt<7>(V);
19568 if (VT.
isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19569 return isShiftedUInt<8, 1>(V);
19572 return isShiftedUInt<8, 2>(V);
19574 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19577 return isUInt<8>(V);
19578 return isUInt<12>(V);
19604 default:
return false;
19609 return isUInt<12>(V);
19612 return isUInt<8>(V);
19617 return isShiftedUInt<8, 2>(V);
19623 int Scale = AM.
Scale;
19628 default:
return false;
19636 Scale = Scale & ~1;
19637 return Scale == 2 || Scale == 4 || Scale == 8;
19654 if (Scale & 1)
return false;
19661 const int Scale = AM.
Scale;
19671 return (Scale == 1) || (!AM.
HasBaseReg && Scale == 2);
19687 switch (AM.
Scale) {
19704 int Scale = AM.
Scale;
19706 default:
return false;
19710 if (Scale < 0) Scale = -Scale;
19718 if (Scale == 1 || (AM.
HasBaseReg && Scale == -1))
19731 if (Scale & 1)
return false;
19744 if (!Subtarget->isThumb())
19751 return Imm >= 0 && Imm <= 255;
19760 int64_t AbsImm = std::abs(Imm);
19761 if (!Subtarget->isThumb())
19766 return AbsImm >= 0 && AbsImm <= 255;
19801 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
19805 int RHSC = (int)
RHS->getZExtValue();
19806 if (RHSC < 0 && RHSC > -256) {
19816 }
else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
19819 int RHSC = (int)
RHS->getZExtValue();
19820 if (RHSC < 0 && RHSC > -0x1000) {
19862 int RHSC = (int)
RHS->getZExtValue();
19863 if (RHSC < 0 && RHSC > -0x100) {
19868 }
else if (RHSC > 0 && RHSC < 0x100) {
19879 bool isSEXTLoad,
bool IsMasked,
bool isLE,
19884 if (!isa<ConstantSDNode>(
Ptr->getOperand(1)))
19890 bool CanChangeType = isLE && !IsMasked;
19893 int RHSC = (int)
RHS->getZExtValue();
19895 auto IsInRange = [&](
int RHSC,
int Limit,
int Scale) {
19896 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
19901 }
else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
19912 if (VT == MVT::v4i16) {
19913 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
19915 }
else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
19916 if (IsInRange(RHSC, 0x80, 1))
19918 }
else if (Alignment >= 4 &&
19919 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
19920 IsInRange(RHSC, 0x80, 4))
19922 else if (Alignment >= 2 &&
19923 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
19924 IsInRange(RHSC, 0x80, 2))
19926 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
19945 bool isSEXTLoad =
false;
19946 bool IsMasked =
false;
19948 Ptr = LD->getBasePtr();
19949 VT = LD->getMemoryVT();
19950 Alignment = LD->getAlign();
19952 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
19953 Ptr = ST->getBasePtr();
19954 VT = ST->getMemoryVT();
19955 Alignment = ST->getAlign();
19957 Ptr = LD->getBasePtr();
19958 VT = LD->getMemoryVT();
19959 Alignment = LD->getAlign();
19963 Ptr = ST->getBasePtr();
19964 VT = ST->getMemoryVT();
19965 Alignment = ST->getAlign();
19971 bool isLegal =
false;
19973 isLegal = Subtarget->hasMVEIntegerOps() &&
19975 Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
20003 bool isSEXTLoad =
false, isNonExt;
20004 bool IsMasked =
false;
20006 VT = LD->getMemoryVT();
20007 Ptr = LD->getBasePtr();
20008 Alignment = LD->getAlign();
20011 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20012 VT = ST->getMemoryVT();
20013 Ptr = ST->getBasePtr();
20014 Alignment = ST->getAlign();
20015 isNonExt = !ST->isTruncatingStore();
20017 VT = LD->getMemoryVT();
20018 Ptr = LD->getBasePtr();
20019 Alignment = LD->getAlign();
20024 VT = ST->getMemoryVT();
20025 Ptr = ST->getBasePtr();
20026 Alignment = ST->getAlign();
20027 isNonExt = !ST->isTruncatingStore();
20035 assert(
Op->getValueType(0) == MVT::i32 &&
"Non-i32 post-inc op?!");
20036 if (
Op->getOpcode() !=
ISD::ADD || !isNonExt)
20038 auto *
RHS = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
20039 if (!
RHS ||
RHS->getZExtValue() != 4)
20041 if (Alignment <
Align(4))
20045 Base =
Op->getOperand(0);
20051 bool isLegal =
false;
20053 isLegal = Subtarget->hasMVEIntegerOps() &&
20086 const APInt &DemandedElts,
20088 unsigned Depth)
const {
20091 switch (
Op.getOpcode()) {
20098 if (
Op.getResNo() == 0) {
20124 case Intrinsic::arm_ldaex:
20125 case Intrinsic::arm_ldrex: {
20126 EVT VT = cast<MemIntrinsicSDNode>(
Op)->getMemoryVT();
20140 const APInt &Mask =
Op.getConstantOperandAPInt(2);
20141 Known.
Zero &= Mask;
20147 const SDValue &SrcSV =
Op.getOperand(0);
20153 "VGETLANE index out of bounds");
20158 EVT VT =
Op.getValueType();
20165 Known = Known.
sext(DstSz);
20167 Known = Known.
zext(DstSz);
20175 Known = KnownOp.
zext(32);
20215 EVT VT =
Op.getValueType();
20221 assert(VT == MVT::i32 &&
"Unexpected integer type");
20228 unsigned Mask =
C->getZExtValue();
20231 unsigned ShrunkMask = Mask & Demanded;
20232 unsigned ExpandedMask = Mask | ~Demanded;
20236 if (ShrunkMask == 0)
20242 if (ExpandedMask == ~0U)
20245 auto IsLegalMask = [ShrunkMask, ExpandedMask](
unsigned Mask) ->
bool {
20246 return (ShrunkMask & Mask) == ShrunkMask && (~ExpandedMask & Mask) == 0;
20248 auto UseMask = [Mask,
Op, VT, &TLO](
unsigned NewMask) ->
bool {
20249 if (NewMask == Mask)
20258 if (IsLegalMask(0xFF))
20259 return UseMask(0xFF);
20262 if (IsLegalMask(0xFFFF))
20263 return UseMask(0xFFFF);
20267 if (ShrunkMask < 256)
20268 return UseMask(ShrunkMask);
20272 if ((
int)ExpandedMask <= -2 && (
int)ExpandedMask >= -256)
20273 return UseMask(ExpandedMask);
20288 unsigned Depth)
const {
20289 unsigned Opc =
Op.getOpcode();
20297 if (
Op.getResNo() == 0 && !
Op->hasAnyUseOfValue(1) &&
20298 isa<ConstantSDNode>(
Op->getOperand(2))) {
20299 unsigned ShAmt =
Op->getConstantOperandVal(2);
20311 unsigned ModImm =
Op.getConstantOperandVal(1);
20312 unsigned EltBits = 0;
20314 if ((OriginalDemandedBits & Mask) == 0)
20320 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO,
Depth);
20329 if (!Subtarget->hasV6Ops())
20335 SplitString(AsmStr, AsmPieces,
";\n");
20337 switch (AsmPieces.
size()) {
20338 default:
return false;
20340 AsmStr = AsmPieces[0];
20342 SplitString(AsmStr, AsmPieces,
" \t,");
20345 if (AsmPieces.
size() == 3 &&
20346 AsmPieces[0] ==
"rev" && AsmPieces[1] ==
"$0" && AsmPieces[2] ==
"$1" &&
20347 IA->getConstraintString().compare(0, 4,
"=l,l") == 0) {
20370 if (ConstraintVT.
isVector() && Subtarget->hasNEON() &&
20382 unsigned S = Constraint.
size();
20384 switch (Constraint[0]) {
20396 }
else if (S == 2) {
20397 switch (Constraint[0]) {
20414 Value *CallOperandVal =
info.CallOperandVal;
20417 if (!CallOperandVal)
20421 switch (*constraint) {
20427 if (Subtarget->isThumb())
20441using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20445 switch (Constraint.
size()) {
20448 switch (Constraint[0]) {
20450 if (Subtarget->isThumb())
20451 return RCPair(0U, &ARM::tGPRRegClass);
20452 return RCPair(0U, &ARM::GPRRegClass);
20454 if (Subtarget->isThumb())
20455 return RCPair(0U, &ARM::hGPRRegClass);
20459 return RCPair(0U, &ARM::tGPRRegClass);
20460 return RCPair(0U, &ARM::GPRRegClass);
20462 if (VT == MVT::Other)
20464 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20465 return RCPair(0U, &ARM::SPRRegClass);
20467 return RCPair(0U, &ARM::DPRRegClass);
20469 return RCPair(0U, &ARM::QPRRegClass);
20472 if (VT == MVT::Other)
20474 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20475 return RCPair(0U, &ARM::SPR_8RegClass);
20477 return RCPair(0U, &ARM::DPR_8RegClass);
20479 return RCPair(0U, &ARM::QPR_8RegClass);
20482 if (VT == MVT::Other)
20484 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20485 return RCPair(0U, &ARM::SPRRegClass);
20487 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20489 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20495 if (Constraint[0] ==
'T') {
20496 switch (Constraint[1]) {
20500 return RCPair(0U, &ARM::tGPREvenRegClass);
20502 return RCPair(0U, &ARM::tGPROddRegClass);
20511 if (
StringRef(
"{cc}").equals_insensitive(Constraint))
20512 return std::make_pair(
unsigned(ARM::CPSR), &ARM::CCRRegClass);
20521 std::vector<SDValue> &Ops,
20526 if (Constraint.
size() != 1)
20529 char ConstraintLetter = Constraint[0];
20530 switch (ConstraintLetter) {
20533 case 'I':
case 'J':
case 'K':
case 'L':
20534 case 'M':
case 'N':
case 'O':
20539 int64_t CVal64 =
C->getSExtValue();
20540 int CVal = (int) CVal64;
20543 if (CVal != CVal64)
20546 switch (ConstraintLetter) {
20550 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
20551 if (CVal >= 0 && CVal <= 65535)
20558 if (CVal >= 0 && CVal <= 255)
20560 }
else if (Subtarget->
isThumb2()) {
20579 if (CVal >= -255 && CVal <= -1)
20585 if (CVal >= -4095 && CVal <= 4095)
20598 }
else if (Subtarget->
isThumb2()) {
20621 if (CVal >= -7 && CVal < 7)
20623 }
else if (Subtarget->
isThumb2()) {
20646 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
20652 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
20660 if (CVal >= 0 && CVal <= 31)
20669 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
20678 if (Result.getNode()) {
20679 Ops.push_back(Result);
20689 "Unhandled Opcode in getDivRemLibcall");
20695 case MVT::i8: LC =
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
20696 case MVT::i16: LC =
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
20697 case MVT::i32: LC =
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
20698 case MVT::i64: LC =
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
20707 "Unhandled Opcode in getDivRemArgList");
20712 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
20713 EVT ArgVT =
N->getOperand(i).getValueType();
20715 Entry.Node =
N->getOperand(i);
20719 Args.push_back(Entry);
20730 "Register-based DivRem lowering only");
20731 unsigned Opcode =
Op->getOpcode();
20733 "Invalid opcode for Div/Rem lowering");
20735 EVT VT =
Op->getValueType(0);
20738 if (VT == MVT::i64 && isa<ConstantSDNode>(
Op.getOperand(1))) {
20757 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
20758 : Subtarget->hasDivideInARMMode();
20759 if (hasDivide &&
Op->getValueType(0).isSimple() &&
20760 Op->getSimpleValueType(0) == MVT::i32) {
20762 const SDValue Dividend =
Op->getOperand(0);
20763 const SDValue Divisor =
Op->getOperand(1);
20764 SDValue Div = DAG.
getNode(DivOpcode, dl, VT, Dividend, Divisor);
20768 SDValue Values[2] = {Div, Rem};
20800 EVT VT =
N->getValueType(0);
20802 if (VT == MVT::i64 && isa<ConstantSDNode>(
N->getOperand(1))) {
20806 Result[0], Result[1]);
20810 std::vector<Type*> RetTyParams;
20811 Type *RetTyElement;
20821 RetTyParams.push_back(RetTyElement);
20822 RetTyParams.push_back(RetTyElement);
20839 CallLoweringInfo CLI(DAG);
20843 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
20846 SDNode *ResNode = CallResult.first.getNode();
20861 "no-stack-arg-probe")) {
20863 cast<ConstantSDNode>(
Op.getOperand(2))->getMaybeAlignValue();
20872 SDValue Ops[2] = { SP, Chain };
20889 SDValue Ops[2] = { NewSP, Chain };
20894 bool IsStrict =
Op->isStrictFPOpcode();
20895 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20896 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20898 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
20899 "Unexpected type for custom-lowering FP_EXTEND");
20902 "With both FP DP and 16, any FP conversion is legal!");
20904 assert(!(DstSz == 32 && Subtarget->hasFP16()) &&
20905 "With FP16, 16 to 32 conversion is legal!");
20908 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) {
20913 Loc,
Op.getValueType(), SrcVal);
20926 MakeLibCallOptions CallOptions;
20928 for (
unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
20929 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
20930 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
20931 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
20935 {DstVT, MVT::Other}, {Chain, SrcVal});
20942 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20943 "Unexpected type for custom-lowering FP_EXTEND");
20944 std::tie(SrcVal, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20949 return IsStrict ? DAG.
getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
20953 bool IsStrict =
Op->isStrictFPOpcode();
20955 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20957 EVT DstVT =
Op.getValueType();
20958 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20961 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
20962 "Unexpected type for custom-lowering FP_ROUND");
20965 "With both FP DP and 16, any FP conversion is legal!");
20970 if (SrcSz == 32 && Subtarget->hasFP16())
20975 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20976 "Unexpected type for custom-lowering FP_ROUND");
20977 MakeLibCallOptions CallOptions;
20980 std::tie(Result, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20992 if (v == 0xffffffff)
21004 bool ForCodeSize)
const {
21007 if (VT == MVT::f16 && Subtarget->hasFullFP16())
21009 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
21012 if (VT == MVT::f32)
21014 if (VT == MVT::f64 && Subtarget->hasFP64())
21025 unsigned Intrinsic)
const {
21026 switch (Intrinsic) {
21027 case Intrinsic::arm_neon_vld1:
21028 case Intrinsic::arm_neon_vld2:
21029 case Intrinsic::arm_neon_vld3:
21030 case Intrinsic::arm_neon_vld4:
21031 case Intrinsic::arm_neon_vld2lane:
21032 case Intrinsic::arm_neon_vld3lane:
21033 case Intrinsic::arm_neon_vld4lane:
21034 case Intrinsic::arm_neon_vld2dup:
21035 case Intrinsic::arm_neon_vld3dup:
21036 case Intrinsic::arm_neon_vld4dup: {
21039 auto &
DL =
I.getDataLayout();
21040 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21042 Info.ptrVal =
I.getArgOperand(0);
21044 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21045 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21050 case Intrinsic::arm_neon_vld1x2:
21051 case Intrinsic::arm_neon_vld1x3:
21052 case Intrinsic::arm_neon_vld1x4: {
21055 auto &
DL =
I.getDataLayout();
21056 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21058 Info.ptrVal =
I.getArgOperand(
I.arg_size() - 1);
21060 Info.align.reset();
21065 case Intrinsic::arm_neon_vst1:
21066 case Intrinsic::arm_neon_vst2:
21067 case Intrinsic::arm_neon_vst3:
21068 case Intrinsic::arm_neon_vst4:
21069 case Intrinsic::arm_neon_vst2lane:
21070 case Intrinsic::arm_neon_vst3lane:
21071 case Intrinsic::arm_neon_vst4lane: {
21074 auto &
DL =
I.getDataLayout();
21075 unsigned NumElts = 0;
21076 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21077 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21080 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21083 Info.ptrVal =
I.getArgOperand(0);
21085 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21086 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21091 case Intrinsic::arm_neon_vst1x2:
21092 case Intrinsic::arm_neon_vst1x3:
21093 case Intrinsic::arm_neon_vst1x4: {
21096 auto &
DL =
I.getDataLayout();
21097 unsigned NumElts = 0;
21098 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21099 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21102 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21105 Info.ptrVal =
I.getArgOperand(0);
21107 Info.align.reset();
21112 case Intrinsic::arm_mve_vld2q:
21113 case Intrinsic::arm_mve_vld4q: {
21116 Type *VecTy = cast<StructType>(
I.getType())->getElementType(1);
21117 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21119 Info.ptrVal =
I.getArgOperand(0);
21126 case Intrinsic::arm_mve_vst2q:
21127 case Intrinsic::arm_mve_vst4q: {
21130 Type *VecTy =
I.getArgOperand(1)->getType();
21131 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21133 Info.ptrVal =
I.getArgOperand(0);
21140 case Intrinsic::arm_mve_vldr_gather_base:
21141 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21143 Info.ptrVal =
nullptr;
21149 case Intrinsic::arm_mve_vldr_gather_base_wb:
21150 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21152 Info.ptrVal =
nullptr;
21158 case Intrinsic::arm_mve_vldr_gather_offset:
21159 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21161 Info.ptrVal =
nullptr;
21163 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
21170 case Intrinsic::arm_mve_vstr_scatter_base:
21171 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21173 Info.ptrVal =
nullptr;
21179 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21180 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21182 Info.ptrVal =
nullptr;
21188 case Intrinsic::arm_mve_vstr_scatter_offset:
21189 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21191 Info.ptrVal =
nullptr;
21193 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(3))->getZExtValue();
21200 case Intrinsic::arm_ldaex:
21201 case Intrinsic::arm_ldrex: {
21202 auto &
DL =
I.getDataLayout();
21203 Type *ValTy =
I.getParamElementType(0);
21206 Info.ptrVal =
I.getArgOperand(0);
21208 Info.align =
DL.getABITypeAlign(ValTy);
21212 case Intrinsic::arm_stlex:
21213 case Intrinsic::arm_strex: {
21214 auto &
DL =
I.getDataLayout();
21215 Type *ValTy =
I.getParamElementType(1);
21218 Info.ptrVal =
I.getArgOperand(1);
21220 Info.align =
DL.getABITypeAlign(ValTy);
21224 case Intrinsic::arm_stlexd:
21225 case Intrinsic::arm_strexd:
21227 Info.memVT = MVT::i64;
21228 Info.ptrVal =
I.getArgOperand(2);
21234 case Intrinsic::arm_ldaexd:
21235 case Intrinsic::arm_ldrexd:
21237 Info.memVT = MVT::i64;
21238 Info.ptrVal =
I.getArgOperand(0);
21258 if (Bits == 0 || Bits > 32)
21264 unsigned Index)
const {
21276 if (!Subtarget->hasDataBarrier()) {
21280 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21317 if (Subtarget->preferISHSTBarriers())
21350 bool has64BitAtomicStore;
21352 has64BitAtomicStore =
false;
21353 else if (Subtarget->isThumb())
21354 has64BitAtomicStore = Subtarget->hasV7Ops();
21356 has64BitAtomicStore = Subtarget->hasV6Ops();
21358 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21372 bool has64BitAtomicLoad;
21374 has64BitAtomicLoad =
false;
21375 else if (Subtarget->isThumb())
21376 has64BitAtomicLoad = Subtarget->hasV7Ops();
21378 has64BitAtomicLoad = Subtarget->hasV6Ops();
21395 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21396 else if (Subtarget->isThumb())
21397 hasAtomicRMW = Subtarget->hasV7Ops();
21399 hasAtomicRMW = Subtarget->hasV6Ops();
21400 if (
Size <= (Subtarget->
isMClass() ? 32U : 64U) && hasAtomicRMW) {
21423 bool HasAtomicCmpXchg;
21425 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21426 else if (Subtarget->isThumb())
21427 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21429 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21431 HasAtomicCmpXchg &&
Size <= (Subtarget->
isMClass() ? 32U : 64U))
21438 return InsertFencesForAtomic;
21451 M.getOrInsertGlobal(
"__security_cookie",
21459 F->addParamAttr(0, Attribute::AttrKind::InReg);
21465 return M.getGlobalVariable(
"__security_cookie");
21472 return M.getFunction(
"__security_check_cookie");
21477 unsigned &
Cost)
const {
21479 if (!Subtarget->hasNEON())
21491 if (!isa<ConstantInt>(
Idx))
21506 return Subtarget->hasV6T2Ops();
21510 return Subtarget->hasV6T2Ops();
21515 if (!Subtarget->hasV7Ops())
21521 if (!Mask || Mask->getValue().getBitWidth() > 32u)
21523 auto MaskVal =
unsigned(Mask->getValue().getZExtValue());
21548 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
21560 Lo, Builder.
CreateShl(
Hi, ConstantInt::get(ValueTy, 32)),
"val64");
21563 Type *Tys[] = {
Addr->getType() };
21564 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
21569 0,
Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
21575 if (!Subtarget->hasV7Ops())
21592 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
21603 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
21604 Type *Tys[] = {
Addr->getType() };
21626 return (
DL.getTypeSizeInBits(VecTy) + 127) / 128;
21633 unsigned VecSize =
DL.getTypeSizeInBits(VecTy);
21636 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
21644 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
21652 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
21655 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
21660 if (Subtarget->hasNEON() && VecSize == 64)
21662 return VecSize % 128 == 0;
21666 if (Subtarget->hasNEON())
21668 if (Subtarget->hasMVEIntegerOps())
21688 "Invalid interleave factor");
21689 assert(!Shuffles.
empty() &&
"Empty shufflevector input");
21691 "Unmatched number of shufflevectors and indices");
21693 auto *VecTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21694 Type *EltTy = VecTy->getElementType();
21717 if (NumLoads > 1) {
21721 VecTy->getNumElements() / NumLoads);
21726 auto createLoadIntrinsic = [&](
Value *BaseAddr) {
21727 if (Subtarget->hasNEON()) {
21729 Type *Tys[] = {VecTy, PtrTy};
21730 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
21731 Intrinsic::arm_neon_vld3,
21732 Intrinsic::arm_neon_vld4};
21740 return Builder.
CreateCall(VldnFunc, Ops,
"vldN");
21742 assert((Factor == 2 || Factor == 4) &&
21743 "expected interleave factor of 2 or 4 for MVE");
21745 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
21747 Type *Tys[] = {VecTy, PtrTy};
21753 return Builder.
CreateCall(VldnFunc, Ops,
"vldN");
21762 for (
unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
21767 VecTy->getNumElements() * Factor);
21769 CallInst *VldN = createLoadIntrinsic(BaseAddr);
21773 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21775 unsigned Index = Indices[i];
21785 SubVecs[SV].push_back(SubVec);
21794 auto &SubVec = SubVecs[SVI];
21797 SVI->replaceAllUsesWith(WideVec);
21831 unsigned Factor)
const {
21833 "Invalid interleave factor");
21835 auto *VecTy = cast<FixedVectorType>(SVI->
getType());
21836 assert(VecTy->getNumElements() % Factor == 0 &&
"Invalid interleaved store");
21838 unsigned LaneLen = VecTy->getNumElements() / Factor;
21839 Type *EltTy = VecTy->getElementType();
21843 Align Alignment = SI->getAlign();
21860 Type *IntTy =
DL.getIntPtrType(EltTy);
21872 Value *BaseAddr = SI->getPointerOperand();
21874 if (NumStores > 1) {
21877 LaneLen /= NumStores;
21885 auto createStoreIntrinsic = [&](
Value *BaseAddr,
21887 if (Subtarget->hasNEON()) {
21888 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
21889 Intrinsic::arm_neon_vst3,
21890 Intrinsic::arm_neon_vst4};
21891 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21892 Type *Tys[] = {PtrTy, SubVecTy};
21895 SI->getModule(), StoreInts[Factor - 2], Tys);
21903 assert((Factor == 2 || Factor == 4) &&
21904 "expected interleave factor of 2 or 4 for MVE");
21906 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
21907 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21908 Type *Tys[] = {PtrTy, SubVecTy};
21915 for (
unsigned F = 0;
F < Factor;
F++) {
21923 for (
unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
21926 if (StoreCount > 0)
21928 BaseAddr, LaneLen * Factor);
21933 for (
unsigned i = 0; i < Factor; i++) {
21934 unsigned IdxI = StoreCount * LaneLen * Factor + i;
21935 if (Mask[IdxI] >= 0) {
21939 unsigned StartMask = 0;
21940 for (
unsigned j = 1; j < LaneLen; j++) {
21941 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
21942 if (Mask[IdxJ * Factor + IdxI] >= 0) {
21943 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
21958 createStoreIntrinsic(BaseAddr, Shuffles);
21973 if (
auto *ST = dyn_cast<StructType>(Ty)) {
21974 for (
unsigned i = 0; i < ST->getNumElements(); ++i) {
21978 Members += SubMembers;
21980 }
else if (
auto *AT = dyn_cast<ArrayType>(Ty)) {
21984 Members += SubMembers * AT->getNumElements();
21995 }
else if (
auto *VT = dyn_cast<VectorType>(Ty)) {
22002 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
22004 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
22006 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
22019 return (Members > 0 && Members <= 4);
22025 const Align ABITypeAlign =
DL.getABITypeAlign(ArgTy);
22027 return ABITypeAlign;
22031 return std::min(ABITypeAlign,
DL.getStackAlignment());
22040 if (getEffectiveCallingConv(CallConv, isVarArg) !=
22050 return IsHA || IsIntArray;
22054 const Constant *PersonalityFn)
const {
22061 const Constant *PersonalityFn)
const {
22073void ARMTargetLowering::insertCopiesSplitCSR(
22077 const MCPhysReg *IStart =
TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
22087 RC = &ARM::GPRRegClass;
22088 else if (ARM::DPRRegClass.
contains(*
I))
22089 RC = &ARM::DPRRegClass;
22099 assert(Entry->getParent()->getFunction().hasFnAttribute(
22100 Attribute::NoUnwind) &&
22101 "Function should be nounwind in insertCopiesSplitCSR!");
22102 Entry->addLiveIn(*
I);
22107 for (
auto *Exit : Exits)
22109 TII->get(TargetOpcode::COPY), *
I)
22120 return Subtarget->hasMVEIntegerOps();
22125 auto *VTy = dyn_cast<FixedVectorType>(Ty);
22130 unsigned NumElements = VTy->getNumElements();
22137 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22138 return Subtarget->hasMVEFloatOps();
22143 return Subtarget->hasMVEIntegerOps() &&
22144 (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
22145 ScalarTy->isIntegerTy(32));
22157 assert(TyWidth >= 128 &&
"Width of vector type must be at least 128 bits");
22159 if (TyWidth > 128) {
22164 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22166 auto *LowerSplitA =
B.CreateShuffleVector(InputA, LowerSplitMask);
22167 auto *LowerSplitB =
B.CreateShuffleVector(InputB, LowerSplitMask);
22168 auto *UpperSplitA =
B.CreateShuffleVector(InputA, UpperSplitMask);
22169 auto *UpperSplitB =
B.CreateShuffleVector(InputB, UpperSplitMask);
22170 Value *LowerSplitAcc =
nullptr;
22171 Value *UpperSplitAcc =
nullptr;
22174 LowerSplitAcc =
B.CreateShuffleVector(
Accumulator, LowerSplitMask);
22175 UpperSplitAcc =
B.CreateShuffleVector(
Accumulator, UpperSplitMask);
22179 B, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
22181 B, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);
22184 return B.CreateShuffleVector(LowerSplitInt, UpperSplitInt, JoinMask);
22191 ConstRotation = ConstantInt::get(IntTy, (
int)Rotation);
22194 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmlaq, Ty,
22196 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmulq, Ty,
22197 {ConstRotation, InputB, InputA});
22202 auto *ConstHalving = ConstantInt::get(IntTy, 1);
22205 ConstRotation = ConstantInt::get(IntTy, 0);
22207 ConstRotation = ConstantInt::get(IntTy, 1);
22209 if (!ConstRotation)
22212 return B.CreateIntrinsic(Intrinsic::arm_mve_vcaddq, Ty,
22213 {ConstHalving, ConstRotation, InputA, InputB});
unsigned const MachineRegisterInfo * MRI
static bool isAddSubSExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG, bool isSigned)
static bool isZeroExtended(SDValue N, SelectionDAG &DAG)
static bool areExtractExts(Value *Ext1, Value *Ext2)
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static const MCPhysReg GPRArgRegs[]
static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1, SDValue V2, unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
static bool isSignExtended(SDValue N, SelectionDAG &DAG)
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
static bool isAddSubZExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG)
static bool isStore(int Opcode)
static bool isThumb(const MCSubtargetInfo &STI)
static SDValue LowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, const TargetInstrInfo *TII)
MatchingStackOffset - Return true if the given stack call argument is already available in the same p...
static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG)
static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total value size to 64 bits.
static cl::opt< unsigned > ConstpoolPromotionMaxSize("arm-promote-constant-max-size", cl::Hidden, cl::desc("Maximum size of constant to promote into a constant pool"), cl::init(64))
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isVTBLMask(ArrayRef< int > M, EVT VT)
static SDValue PerformSUBCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
static cl::opt< bool > EnableConstpoolPromotion("arm-promote-constant", cl::Hidden, cl::desc("Enable / disable promotion of unnamed_addr constants into " "constant pools"), cl::init(false))
static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG)
static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
static SDValue PerformExtractEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static const APInt * isPowerOf2Constant(SDValue V)
static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG)
static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static bool isValidMVECond(unsigned CC, bool IsFloat)
static SDValue PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC)
IntCCToARMCC - Convert a DAG integer condition code to an ARM CC.
static SDValue PerformSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSTORECombine - Target-specific dag combine xforms for ISD::STORE.
static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry, SelectionDAG &DAG)
static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isGTorGE(ISD::CondCode CC)
static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a vldN-lane (N > 1) intrinsic,...
static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask)
static bool isReverseMask(ArrayRef< int > M, EVT VT)
static bool isVZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of "vector_shuffle v,...
static SDValue PerformSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVMulVCTPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVMulVCTPCombine - VCVT (fixed-point to floating-point, Advanced SIMD) can replace combinations...
static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG)
static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc)
static bool isVTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool CanInvertMVEVCMP(SDValue N)
static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG)
static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformShiftCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
PerformShiftCombine - Checks for immediate versions of vector shifts and lowers them.
static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, ARMCC::CondCodes &CondCode2)
FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static EVT getVectorTyFromPredicateVector(EVT VT)
static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG)
static SDValue handleCMSEValue(const SDValue &Value, const ISD::InputArg &Arg, SelectionDAG &DAG, const SDLoc &DL)
static SDValue PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
static bool isSRL16(const SDValue &Op)
static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC)
static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr, SDValue Inc, const SelectionDAG &DAG)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static Register genTPEntry(MachineBasicBlock *TpEntry, MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpExit, Register OpSizeReg, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI)
Adds logic in loop entry MBB to calculate loop iteration count and adds t2WhileLoopSetup and t2WhileL...
static bool isLTorLE(ISD::CondCode CC)
static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl, SelectionDAG &DAG)
static SDValue PerformBITCASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG)
static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG)
static bool hasNormalLoadOperand(SDNode *N)
hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node are normal,...
static SDValue PerformInsertEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
PerformInsertEltCombine - Target-specific dag combine xforms for ISD::INSERT_VECTOR_ELT.
static SDValue PerformVDUPLANECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVDUPLANECombine - Target-specific dag combine xforms for ARMISD::VDUPLANE.
static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static cl::opt< unsigned > ConstpoolPromotionMaxTotal("arm-promote-constant-max-total", cl::Hidden, cl::desc("Maximum size of ALL constants to promote into a constant pool"), cl::init(128))
static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static RTLIB::Libcall getDivRemLibcall(const SDNode *N, MVT::SimpleValueType SVT)
static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG &DAG)
SkipLoadExtensionForVMULL - return a load of the original vector size that does not do any sign/zero ...
static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static const MCPhysReg GPRArgRegs[]
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombineWithOperands - Try DAG combinations for an ADD with operands N0 and N1.
static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT, SelectionDAG &DAG)
static bool isVZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformORCombineToSMULWBT(SDNode *OR, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isVTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of "vector_shuffle v,...
static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue FindBFIToCombineWith(SDNode *N)
static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT, SelectionDAG &DAG)
static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, bool &swpCmpOps, bool &swpVselOps)
static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isS16(const SDValue &Op, SelectionDAG &DAG)
static bool isSRA16(const SDValue &Op)
static SDValue AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue LowerInterruptReturn(SmallVectorImpl< SDValue > &RetOps, const SDLoc &DL, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl, SelectionDAG &DAG)
static void expandf64Toi32(SDValue Op, SelectionDAG &DAG, SDValue &RetVal1, SDValue &RetVal2)
static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isSHL16(const SDValue &Op)
static bool isVEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseVEXT, unsigned &Imm)
static SDValue PerformMVEVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isTruncMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue PerformADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2)
Return the load opcode for a given load size.
static bool isLegalT2AddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
static bool isLegalMVEShuffleOp(unsigned PFEntry)
static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N, SelectionDAG &DAG)
static bool isVUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG)
PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for ISD::VECTOR_SHUFFLE.
static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG)
SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND, ANY_EXTEND,...
static bool isVMOVNTruncMask(ArrayRef< int > M, EVT ToVT, bool rev)
static SDValue PerformVQMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static MachineBasicBlock * OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ)
static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformAddcSubcCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static TargetLowering::ArgListTy getDivRemArgList(const SDNode *N, LLVMContext *Context, const ARMSubtarget *Subtarget)
static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl)
getZeroVector - Returns a vector of specified type with all zero elements.
static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static ARMCC::CondCodes getVCMPCondCode(SDValue N)
static cl::opt< bool > ARMInterworking("arm-interworking", cl::Hidden, cl::desc("Enable / disable ARM interworking (for debugging only)"), cl::init(true))
static void ReplaceREADCYCLECOUNTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformORCombineToBFI(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, SDValue &CC, bool &Invert, SDValue &OtherOp, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVSetCCToVCTPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isZeroVector(SDValue N)
static SDValue PerformAddeSubeCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void ReplaceCMP_SWAP_64Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isLowerSaturate(const SDValue LHS, const SDValue RHS, const SDValue TrueVal, const SDValue FalseVal, const ISD::CondCode CC, const SDValue K)
static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG)
static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned StSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment store operation with given size.
static bool isVMOVNMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue CombineBaseUpdate(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineBaseUpdate - Target-specific DAG combine function for VLDDUP, NEON load/store intrinsics,...
static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVRRDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMOVRRDCombine - Target-specific dag combine xforms for ARMISD::VMOVRRD.
static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain)
static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue PerformVMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMULCombine Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the special multi...
static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG)
static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformORCombine - Target-specific dag combine xforms for ISD::OR.
static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG)
static SDValue PerformTruncatingStoreCombine(StoreSDNode *St, SelectionDAG &DAG)
static unsigned SelectPairHalf(unsigned Elements, ArrayRef< int > Mask, unsigned Index)
static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned LdSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment load operation with given size.
static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG)
static bool isValidBaseUpdate(SDNode *N, SDNode *User)
static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG, const ARMSubtarget *ST, const SDLoc &dl)
static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op)
static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformXORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment, bool isSEXTLoad, bool IsMasked, bool isLE, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
std::pair< unsigned, const TargetRegisterClass * > RCPair
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,...
static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
cl::opt< unsigned > MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden, cl::desc("Maximum interleave factor for MVE VLDn to generate."), cl::init(2))
static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, unsigned SplatBitSize, SelectionDAG &DAG, const SDLoc &dl, EVT &VT, EVT VectorVT, VMOVModImmType type)
isVMOVModifiedImm - Check if the specified splat value corresponds to a valid vector constant for a N...
static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC, SelectionDAG &DAG)
BC is a bitcast that is about to be turned into a VMOVDRR.
static SDValue promoteToConstantPool(const ARMTargetLowering *TLI, const GlobalValue *GV, SelectionDAG &DAG, EVT PtrVT, const SDLoc &dl)
static unsigned isNEONTwoResultShuffleMask(ArrayRef< int > ShuffleMask, EVT VT, unsigned &WhichResult, bool &isV_UNDEF)
Check if ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN), and return the corresponding AR...
static bool BitsProperlyConcatenate(const APInt &A, const APInt &B)
static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG)
static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target, struct BaseUpdateUser &User, bool SimpleConstIncOnly, TargetLowering::DAGCombinerInfo &DCI)
static bool allUsersAreInFunction(const Value *V, const Function *F)
Return true if all users of V are within function F, looking through ConstantExprs.
static bool isSingletonVEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG)
PerformVMOVDRRCombine - Target-specific dag combine xforms for ARMISD::VMOVDRR.
static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, SDValue &SatK)
static bool isLegalAddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
isLegalAddressImmediate - Return true if the integer value can be used as the offset of the target ad...
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isLegalT1AddressImmediate(int64_t V, EVT VT)
static SDValue CombineANDShift(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSHLSimplify(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformADDECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDECombine - Target-specific dag combine transform from ARMISD::ADDC, ARMISD::ADDE,...
static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformHWLoopCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool isVUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of "vector_shuffle v,...
static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base, uint64_t &Members)
static SDValue PerformMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
static SDValue PerformANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG)
static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm, bool &Negate)
static bool canChangeToInt(SDValue Op, bool &SeenZero, const ARMSubtarget *Subtarget)
canChangeToInt - Given the fp compare operand, return true if it is suitable to morph to an integer c...
static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2)
Return the store opcode for a given store size.
static bool IsVUZPShuffleNode(SDNode *N)
static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget, MachineInstr &MI, const SDNode *Node)
Attaches vregs to MEMCPY that it will use as scratch registers when it is expanded into LDM/STM.
static bool isFloatingPointZero(SDValue Op)
isFloatingPointZero - Return true if this is +0.0.
static SDValue findMUL_LOHI(SDValue V)
static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N, SelectionDAG &DAG)
static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG)
static void genTPLoopBody(MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI, Register OpSrcReg, Register OpDestReg, Register ElementCountReg, Register TotalIterationsReg, bool IsMemcpy)
Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and t2DoLoopEnd.
static SDValue PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformBUILD_VECTORCombine - Target-specific dag combine xforms for ISD::BUILD_VECTOR.
static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformMinMaxCombine - Target-specific DAG combining for creating truncating saturates.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static Function * getFunction(Constant *C)
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
static Value * LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctpop of V before the specified instruction IP.
loop Loop Strength Reduction
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
This file describes how to lower LLVM code to machine code.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Assuming the flags are set by MI(a,b), return the condition code if we modify the instructions such t...
static constexpr int Concat[]
bool getExactInverse(APFloat *inv) const
APInt bitcastToAPInt() const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
virtual const ARMBaseRegisterInfo & getRegisterInfo() const =0
const uint32_t * getSjLjDispatchPreservedMask(const MachineFunction &MF) const
const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const override
Code Generation virtual methods...
Register getFrameRegister(const MachineFunction &MF) const override
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
const uint32_t * getTLSCallPreservedMask(const MachineFunction &MF) const
const uint32_t * getThisReturnPreservedMask(const MachineFunction &MF, CallingConv::ID) const
getThisReturnPreservedMask - Returns a call preserved mask specific to the case that 'returned' is on...
static ARMConstantPoolConstant * Create(const Constant *C, unsigned ID)
static ARMConstantPoolMBB * Create(LLVMContext &C, const MachineBasicBlock *mbb, unsigned ID, unsigned char PCAdj)
static ARMConstantPoolSymbol * Create(LLVMContext &C, StringRef s, unsigned ID, unsigned char PCAdj)
ARMConstantPoolValue - ARM specific constantpool value.
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
int getVarArgsFrameIndex() const
int getPromotedConstpoolIncrease() const
SmallPtrSet< const GlobalVariable *, 2 > & getGlobalsPromotedToConstantPool()
void setArgumentStackToRestore(unsigned v)
bool branchTargetEnforcement() const
unsigned createPICLabelUId()
void setPromotedConstpoolIncrease(int Sz)
bool isThumb1OnlyFunction() const
void setArgRegsSaveSize(unsigned s)
bool shouldSignReturnAddress() const
bool isCmseNSEntryFunction() const
void setReturnRegsCount(unsigned s)
void setVarArgsFrameIndex(int Index)
unsigned getArgRegsSaveSize() const
void markGlobalAsPromotedToConstantPool(const GlobalVariable *GV)
Indicate to the backend that GV has had its storage changed to inside a constant pool.
void setIsSplitCSR(bool s)
void setArgumentStackSize(unsigned size)
unsigned getArgumentStackSize() const
bool isTargetMachO() const
bool isTargetAEABI() const
bool supportsTailCall() const
const Triple & getTargetTriple() const
const ARMBaseInstrInfo * getInstrInfo() const override
bool isThumb1Only() const
bool hasFPARMv8Base() const
bool isTargetWindows() const
bool isGVIndirectSymbol(const GlobalValue *GV) const
True if the GV will be accessed via an indirect symbol.
const ARMTargetLowering * getTargetLowering() const override
bool isTargetDarwin() const
const ARMBaseRegisterInfo * getRegisterInfo() const override
bool isTargetAndroid() const
bool isTargetCOFF() const
bool isTargetGNUAEABI() const
bool isTargetWatchOS() const
bool useNEONForSinglePrecisionFP() const
const InstrItineraryData * getInstrItineraryData() const override
getInstrItins - Return the instruction itineraries based on subtarget selection.
bool isTargetWatchABI() const
bool hasAnyDataBarrier() const
bool isTargetDriverKit() const
bool allowsUnalignedMem() const
bool isTargetMuslAEABI() const
bool isTargetLinux() const
unsigned getPrefLoopLogAlignment() const
bool isTargetHardFloat() const
Align getDualLoadStoreAlignment() const
bool isReadOnly(const GlobalValue *GV) const
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
bool shouldInsertFencesForAtomic(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const override
Return the correct alignment for the current calling convention.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
isLegalAddressingMode - Return true if the addressing mode represented by AM is legal for this target...
const ARMSubtarget * getSubtarget() const
bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const
bool isLegalT1ScaledAddressingMode(const AddrMode &AM, EVT VT) const
Returns true if the addressing mode representing by AM is legal for the Thumb1 target,...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPreIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mod...
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
ReplaceNodeResults - Replace the results of node with an illegal result type with new values built ou...
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
bool isLegalAddImmediate(int64_t Imm) const override
isLegalAddImmediate - Return true if the specified immediate is legal add immediate,...
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
SDValue PerformMVETruncCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const override
isFPImmLegal - Returns true if the target can instruction select the specified FP immediate natively.
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
SDValue PerformIntrinsicCombine(SDNode *N, DAGCombinerInfo &DCI) const
PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
bool isDesirableToCommuteXorWithShift(const SDNode *N) const override
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
bool ExpandInlineAsm(CallInst *CI) const override
This hook allows the target to expand an inline asm call to be explicit llvm code if it wants to.
SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const
PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const override
Create the IR node for the given complex deinterleaving operation.
bool isComplexDeinterleavingSupported() const override
Does this target support complex deinterleaving.
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
SDValue PerformMVEExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
getSetCCResultType - Return the value type to use for ISD::SETCC.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool isVarArg) const
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
createFastISel - This method returns a target specific FastISel object, or null if the target does no...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
isShuffleMaskLegal - Targets can use this to indicate that they only support some VECTOR_SHUFFLE oper...
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const override
getRegClassFor - Return the register class that should be used for the specified value type.
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vstN intrinsic.
ARMTargetLowering(const TargetMachine &TM, const ARMSubtarget &STI)
bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const override
Does this target support complex deinterleaving with the given operation and type.
SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const
PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const override
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
Instruction * makeDMB(IRBuilderBase &Builder, ARM_MB::MemBOpt Domain) const
bool isLegalICmpImmediate(int64_t Imm) const override
isLegalICmpImmediate - Return true if the specified immediate is legal icmp immediate,...
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool isVarArg) const
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
allowsMisalignedMemoryAccesses - Returns true if the target allows unaligned memory accesses of the s...
bool isLegalInterleavedAccessType(unsigned Factor, FixedVectorType *VecTy, Align Alignment, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
bool isVectorLoadExtDesirable(SDValue ExtVal) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const override
Return true if the target can combine store(extractelement VectorTy, Idx).
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vldN intrinsic.
bool useSoftFloat() const override
bool alignLoopsWithOptSize() const override
Should loops be aligned even when the function is marked OptSize (but not MinSize).
SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
LowerAsmOperandForConstraint - Lower the specified operand into the Ops vector.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const override
Returns true if an argument of type Ty needs to be passed in a contiguous block of registers in calli...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPostIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mo...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
bool isFloatingPointOperation() const
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
The address of a basic block.
static BranchProbability getZero()
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
CCState - This class holds information needed while lowering arguments and return values.
void getInRegsParamInfo(unsigned InRegsParamRecordIndex, unsigned &BeginReg, unsigned &EndReg) const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
void rewindByValRegsInfo()
unsigned getInRegsParamsProcessed() const
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void addInRegsParamInfo(unsigned RegBegin, unsigned RegEnd)
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
unsigned getInRegsParamsCount() const
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
unsigned getValNo() const
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
Value * getCalledOperand() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
const APFloat & getValueAPF() const
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
Align getStackAlignment() const
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPreferredAlign(const GlobalVariable *GV) const
Returns the preferred alignment of the specified global.
StringRef getPrivateGlobalPrefix() const
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for unsupported feature in backend.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
const GlobalValue * getGlobal() const
bool hasExternalWeakLinkage() const
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
bool isStrongDefinitionForLinker() const
Returns true if this global's definition will be the one chosen by the linker.
@ InternalLinkage
Rename collisions when linking (static functions).
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
TargetInstrInfo overrides.
Common base class shared among various IRBuilders.
Value * CreateZExtOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
BasicBlock * GetInsertBlock() const
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateTruncOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::optional< unsigned > getOperandCycle(unsigned ItinClassIndx, unsigned OperandIdx) const
Return the cycle for the given class and operand.
bool isEmpty() const
Returns true if there are no itineraries.
bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
static bool LowerToByteSwap(CallInst *CI)
Try to replace a call instruction with a call to a bswap intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Describe properties that are true of each instruction in the target description file.
unsigned getSchedClass() const
Return the scheduling class for this instruction.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
bool isEHPad() const
Returns true if the block is a landing pad.
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool canFallThrough()
Return true if the block can implicitly transfer control to the block after it by falling off the end...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
pred_iterator pred_begin()
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void moveAfter(MachineBasicBlock *NewBefore)
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void computeMaxCallFrameSize(MachineFunction &MF, std::vector< MachineBasicBlock::iterator > *FrameSDOps=nullptr)
Computes the maximum size of a callframe.
void setAdjustsStack(bool V)
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool hasVAStart() const
Returns true if the function calls the llvm.va_start intrinsic.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
int getFunctionContextIndex() const
Return the index for the function context object.
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & reset(Property P)
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
const MachineOperand & getOperand(unsigned i) const
unsigned createJumpTableIndex(const std::vector< MachineBasicBlock * > &DestBBs)
createJumpTableIndex - Create a new jump table.
@ EK_Inline
EK_Inline - Jump table entries are emitted inline at their point of use.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
void setIsDef(bool Val=true)
Change a def to a use, or a use to a def.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class is used to represent an MLOAD node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
virtual void print(raw_ostream &OS, const Module *M) const
print - Print out the internal state of the pass.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< use_iterator > uses()
size_t use_size() const
Return the number of uses of this node.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
void setFlags(SDNodeFlags NewFlags)
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
void addCallSiteInfo(const SDNode *Node, CallSiteInfo &&CallInfo)
Set CallSiteInfo to be associated with Node.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
void reserve(size_type NumEntries)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
StringRef - Represent a constant reference to a string, i.e.
const unsigned char * bytes_end() const
constexpr size_t size() const
size - Get the string size.
const unsigned char * bytes_begin() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
void setCmpLibcallCC(RTLIB::Libcall Call, ISD::CondCode CC)
Override the default CondCode to be used to test the result of the comparison libcall against zero.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
void setMinStackArgumentAlignment(Align Alignment)
Set the minimum stack alignment of an argument.
const TargetMachine & getTargetMachine() const
void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
virtual unsigned getMaxSupportedInterleaveFactor() const
Get the maximum supported factor for interleaved memory accesses.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
bool isPositionIndependent() const
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
bool isOSVersionLT(unsigned Major, unsigned Minor=0, unsigned Micro=0) const
Helper function for doing comparisons against version numbers included in the target triple.
bool isWindowsMSVCEnvironment() const
Checks if the environment could be MSVC.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isArrayTy() const
True if this is an instance of ArrayType.
bool isPointerTy() const
True if this is an instance of PointerType.
Type * getArrayElementType() const
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static CondCodes getOppositeCondition(CondCodes CC)
@ SECREL
Thread Pointer Offset.
@ SBREL
Section Relative (Windows TLS)
@ GOTTPOFF
Global Offset Table, PC Relative.
@ TPOFF
Global Offset Table, Thread Pointer Offset.
TOF
Target Operand Flag enum.
@ MO_NONLAZY
MO_NONLAZY - This is an independent flag, on a symbol operand "FOO" it represents a symbol which,...
@ MO_SBREL
MO_SBREL - On a symbol operand, this represents a static base relative relocation.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_GOT
MO_GOT - On a symbol operand, this represents a GOT relative relocation.
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
unsigned getAM2Offset(unsigned AM2Opc)
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned createVMOVModImm(unsigned OpCmode, unsigned Val)
int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
int getFP32FP16Imm(const APInt &Imm)
If this is a FP16Imm encoded as a fp32 value, return the 8-bit encoding for it.
AddrOpc getAM2Op(unsigned AM2Opc)
bool isBitFieldInvertedMask(unsigned v)
const unsigned FPStatusBits
const unsigned FPReservedBits
const unsigned RoundingBitsPos
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Swift
Calling convention for Swift.
@ ARM_APCS
ARM Procedure Calling Standard (obsolete, but still used on some targets).
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ ARM_AAPCS
ARM Architecture Procedure Calling Standard calling convention (aka EABI).
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool CC_ARM_APCS_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool RetCC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns true if Val1 has a lower Constant Materialization Cost than Val2.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool FastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool RetCC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
void shuffle(Iterator first, Iterator last, RNG &&g)
static std::array< MachineOperand, 2 > predOps(ARMCC::CondCodes Pred, unsigned PredReg=0)
Get the operands corresponding to the given Pred value.
bool CC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
bool CC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_Win32_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
ComplexDeinterleavingOperation
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
ComplexDeinterleavingRotation
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
static MachineOperand t1CondCodeOp(bool isDead=false)
Get the operand corresponding to the conditional code result for Thumb1.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
static MachineOperand condCodeOp(unsigned CCReg=0)
Get the operand corresponding to the conditional code result.
bool isVREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isVREVMask - Check if a vector shuffle corresponds to a VREV instruction with the specified blocksize...
unsigned gettBLXrOpcode(const MachineFunction &MF)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
unsigned convertAddSubFlagsOpcode(unsigned OldOpc)
Map pseudo instructions that imply an 'S' bit onto real opcodes.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
static const unsigned PerfectShuffleTable[6561+1]
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Load/store instruction that can be merged with a base address update.
SDNode * N
Instruction that updates a pointer.
unsigned ConstInc
Pointer increment value if it is a constant, or 0 otherwise.
SDValue Inc
Pointer increment operand.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
static constexpr roundingMode rmTowardZero
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
bool isUnknown() const
Returns true if we don't know any bits.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false)
Compute knownbits resulting from addition of LHS and RHS.
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setInRegister(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)