84#include "llvm/IR/IntrinsicsARM.h"
120#define DEBUG_TYPE "arm-isel"
123STATISTIC(NumMovwMovt,
"Number of GAs materialized with movw + movt");
124STATISTIC(NumLoopByVals,
"Number of loops generated for byval arguments");
126 "Number of constants with their storage promoted into constant pools");
130 cl::desc(
"Enable / disable ARM interworking (for debugging only)"),
135 cl::desc(
"Enable / disable promotion of unnamed_addr constants into "
140 cl::desc(
"Maximum size of constant to promote into a constant pool"),
144 cl::desc(
"Maximum size of ALL constants to promote into a constant pool"),
149 cl::desc(
"Maximum interleave factor for MVE VLDn to generate."),
157 ARM::R0, ARM::R1, ARM::R2, ARM::R3
171void ARMTargetLowering::addTypeForNEON(
MVT VT,
MVT PromotedLdStVT) {
172 if (VT != PromotedLdStVT) {
181 if (ElemTy != MVT::f64)
185 if (ElemTy == MVT::i32) {
229void ARMTargetLowering::addDRTypeForNEON(
MVT VT) {
231 addTypeForNEON(VT, MVT::f64);
234void ARMTargetLowering::addQRTypeForNEON(
MVT VT) {
236 addTypeForNEON(VT, MVT::v2f64);
239void ARMTargetLowering::setAllExpand(
MVT VT) {
252void ARMTargetLowering::addAllExtLoads(
const MVT From,
const MVT To,
253 LegalizeAction Action) {
259void ARMTargetLowering::addMVEVectorTypes(
bool HasMVEFP) {
260 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
262 for (
auto VT : IntTypes) {
336 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
337 for (
auto VT : FloatTypes) {
406 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
407 for (
auto VT : LongTypes) {
424 addAllExtLoads(MVT::v8i16, MVT::v8i8,
Legal);
425 addAllExtLoads(MVT::v4i32, MVT::v4i16,
Legal);
426 addAllExtLoads(MVT::v4i32, MVT::v4i8,
Legal);
443 for (
auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
452 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
453 for (
auto VT : pTypes) {
509 for (
int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID)
517 if (Subtarget->isThumb() && Subtarget->
hasVFP2Base() &&
518 Subtarget->
hasARMOps() && !Subtarget->useSoftFloat()) {
519 static const struct {
521 const char *
const Name;
543 { RTLIB::UO_F32,
"__unordsf2vfp",
ISD::SETNE },
552 { RTLIB::UO_F64,
"__unorddf2vfp",
ISD::SETNE },
577 for (
const auto &LC : LibraryCalls) {
589 static const struct {
591 const char *
const Name;
676 for (
const auto &LC : LibraryCalls) {
686 static const struct {
688 const char *
const Name;
691 } MemOpsLibraryCalls[] = {
699 for (
const auto &LC : MemOpsLibraryCalls) {
709 static const struct {
711 const char *
const Name;
724 for (
const auto &LC : LibraryCalls) {
756 static const struct {
758 const char *
const Name;
766 for (
const auto &LC : LibraryCalls) {
777 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only() &&
778 Subtarget->hasFPRegs()) {
788 setAllExpand(MVT::f32);
789 if (!Subtarget->hasFP64())
790 setAllExpand(MVT::f64);
793 if (Subtarget->hasFullFP16()) {
802 if (Subtarget->hasBF16()) {
804 setAllExpand(MVT::bf16);
805 if (!Subtarget->hasFullFP16())
817 addAllExtLoads(VT, InnerVT,
Expand);
832 if (Subtarget->hasMVEIntegerOps())
833 addMVEVectorTypes(Subtarget->hasMVEFloatOps());
836 if (Subtarget->hasLOB()) {
840 if (Subtarget->hasNEON()) {
841 addDRTypeForNEON(MVT::v2f32);
842 addDRTypeForNEON(MVT::v8i8);
843 addDRTypeForNEON(MVT::v4i16);
844 addDRTypeForNEON(MVT::v2i32);
845 addDRTypeForNEON(MVT::v1i64);
847 addQRTypeForNEON(MVT::v4f32);
848 addQRTypeForNEON(MVT::v2f64);
849 addQRTypeForNEON(MVT::v16i8);
850 addQRTypeForNEON(MVT::v8i16);
851 addQRTypeForNEON(MVT::v4i32);
852 addQRTypeForNEON(MVT::v2i64);
854 if (Subtarget->hasFullFP16()) {
855 addQRTypeForNEON(MVT::v8f16);
856 addDRTypeForNEON(MVT::v4f16);
859 if (Subtarget->hasBF16()) {
860 addQRTypeForNEON(MVT::v8bf16);
861 addDRTypeForNEON(MVT::v4bf16);
865 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
905 if (Subtarget->hasNEON()) {
1019 for (
MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
1028 for (
auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
1037 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
1045 if (Subtarget->hasMVEIntegerOps()) {
1050 if (Subtarget->hasMVEFloatOps()) {
1054 if (!Subtarget->hasFP64()) {
1101 if (Subtarget->hasFullFP16()) {
1107 if (!Subtarget->hasFP16()) {
1158 if (Subtarget->hasDSP()) {
1180 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6Ops()
1181 || (Subtarget->
isThumb2() && !Subtarget->hasDSP()))
1196 if (Subtarget->hasMVEIntegerOps())
1206 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops())
1217 if (!Subtarget->hasV5TOps() || Subtarget->
isThumb1Only()) {
1226 if (Subtarget->hasPerfMon())
1230 if (!Subtarget->hasV6Ops())
1233 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1234 : Subtarget->hasDivideInARMMode();
1241 if (Subtarget->
isTargetWindows() && !Subtarget->hasDivideInThumbMode()) {
1258 HasStandaloneRem =
false;
1263 const char *
const Name;
1265 } LibraryCalls[] = {
1277 for (
const auto &LC : LibraryCalls) {
1284 const char *
const Name;
1286 } LibraryCalls[] = {
1298 for (
const auto &LC : LibraryCalls) {
1336 InsertFencesForAtomic =
false;
1338 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
1342 if (!Subtarget->isThumb() || !Subtarget->
isMClass())
1347 if (!Subtarget->hasAcquireRelease() ||
1350 InsertFencesForAtomic =
true;
1356 if (Subtarget->hasDataBarrier())
1357 InsertFencesForAtomic =
true;
1377 if (!InsertFencesForAtomic) {
1385 (!Subtarget->
isMClass() && Subtarget->hasV6Ops())) {
1397 }
else if ((Subtarget->
isMClass() && Subtarget->hasV8MBaselineOps()) ||
1398 Subtarget->hasForced32BitAtomics()) {
1412 if (!Subtarget->hasV6Ops()) {
1418 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1450 if (Subtarget->hasFullFP16()) {
1460 if (Subtarget->hasFullFP16())
1475 if (!Subtarget->useSoftFloat() && Subtarget->
hasVFP2Base() &&
1489 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only()) {
1497 if (!Subtarget->hasFP16()) {
1528 if (Subtarget->hasNEON()) {
1535 if (Subtarget->hasFP64()) {
1548 if (Subtarget->hasFullFP16()) {
1567 if (Subtarget->hasNEON()) {
1579 if (Subtarget->hasFullFP16()) {
1611 if (Subtarget->hasMVEIntegerOps())
1614 if (Subtarget->hasV6Ops())
1619 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1626 if (Subtarget->useSoftFloat() || Subtarget->
isThumb1Only() ||
1655 return Subtarget->useSoftFloat();
1668std::pair<const TargetRegisterClass *, uint8_t>
1679 case MVT::f32:
case MVT::f64:
case MVT::v8i8:
case MVT::v4i16:
1680 case MVT::v2i32:
case MVT::v1i64:
case MVT::v2f32:
1681 RRC = &ARM::DPRRegClass;
1689 case MVT::v16i8:
case MVT::v8i16:
case MVT::v4i32:
case MVT::v2i64:
1690 case MVT::v4f32:
case MVT::v2f64:
1691 RRC = &ARM::DPRRegClass;
1695 RRC = &ARM::DPRRegClass;
1699 RRC = &ARM::DPRRegClass;
1703 return std::make_pair(RRC,
Cost);
1707#define MAKE_CASE(V) \
1926 if ((Subtarget->hasMVEIntegerOps() &&
1927 (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
1928 VT == MVT::v16i8)) ||
1929 (Subtarget->hasMVEFloatOps() &&
1930 (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16)))
1944 if (Subtarget->hasNEON()) {
1945 if (VT == MVT::v4i64)
1946 return &ARM::QQPRRegClass;
1947 if (VT == MVT::v8i64)
1948 return &ARM::QQQQPRRegClass;
1950 if (Subtarget->hasMVEIntegerOps()) {
1951 if (VT == MVT::v4i64)
1952 return &ARM::MQQPRRegClass;
1953 if (VT == MVT::v8i64)
1954 return &ARM::MQQQQPRRegClass;
1963 Align &PrefAlign)
const {
1964 if (!isa<MemIntrinsic>(CI))
1982 unsigned NumVals =
N->getNumValues();
1986 for (
unsigned i = 0; i != NumVals; ++i) {
1987 EVT VT =
N->getValueType(i);
1988 if (VT == MVT::Glue || VT == MVT::Other)
1994 if (!
N->isMachineOpcode())
2018 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2019 return Const->getZExtValue() == 16;
2026 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2027 return Const->getZExtValue() == 16;
2034 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2035 return Const->getZExtValue() == 16;
2104 bool isVarArg)
const {
2125 else if (Subtarget->hasFPRegs() && !Subtarget->
isThumb1Only() &&
2146 bool isVarArg)
const {
2147 return CCAssignFnForNode(
CC,
false, isVarArg);
2151 bool isVarArg)
const {
2152 return CCAssignFnForNode(
CC,
true, isVarArg);
2159 bool isVarArg)
const {
2160 switch (getEffectiveCallingConv(
CC, isVarArg)) {
2186 if (Subtarget->hasFullFP16()) {
2199 if (Subtarget->hasFullFP16()) {
2213SDValue ARMTargetLowering::LowerCallResult(
2217 SDValue ThisVal,
bool isCmseNSCall)
const {
2225 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
2230 if (i == 0 && isThisReturn) {
2232 "unexpected return calling convention register assignment");
2243 Chain =
Lo.getValue(1);
2244 InGlue =
Lo.getValue(2);
2248 Chain =
Hi.getValue(1);
2249 InGlue =
Hi.getValue(2);
2261 Chain =
Lo.getValue(1);
2262 InGlue =
Lo.getValue(2);
2265 Chain =
Hi.getValue(1);
2266 InGlue =
Hi.getValue(2);
2310std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
2312 bool IsTailCall,
int SPDiff)
const {
2334 return std::make_pair(DstAddr, DstInfo);
2343ARMTargetLowering::ByValCopyKind ARMTargetLowering::ByValNeedsCopyForTailCall(
2349 if (isa<GlobalAddressSDNode>(Src) || isa<ExternalSymbolSDNode>(Src))
2354 auto *SrcFrameIdxNode = dyn_cast<FrameIndexSDNode>(Src);
2355 auto *DstFrameIdxNode = dyn_cast<FrameIndexSDNode>(Dst);
2356 if (!SrcFrameIdxNode || !DstFrameIdxNode)
2359 int SrcFI = SrcFrameIdxNode->getIndex();
2360 int DstFI = DstFrameIdxNode->getIndex();
2362 "byval passed in non-fixed stack slot");
2384 if (SrcOffset == DstOffset)
2392 RegsToPassVector &RegsToPass,
2399 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
2400 unsigned id = Subtarget->
isLittle() ? 0 : 1;
2413 std::tie(DstAddr, DstInfo) =
2414 computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
2447 bool isStructRet = (Outs.
empty()) ?
false : Outs[0].
Flags.isSRet();
2448 bool isThisReturn =
false;
2449 bool isCmseNSCall =
false;
2450 bool isSibCall =
false;
2451 bool PreferIndirect =
false;
2452 bool GuardWithBTI =
false;
2462 !Subtarget->noBTIAtReturnTwice())
2467 isCmseNSCall =
true;
2479 if (isa<GlobalAddressSDNode>(Callee)) {
2483 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
2486 PreferIndirect = Subtarget->isThumb() && Subtarget->
hasMinSize() &&
2488 return isa<Instruction>(U) &&
2489 cast<Instruction>(U)->getParent() == BB;
2496 IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs, PreferIndirect);
2510 "site marked musttail");
2513 unsigned NumBytes = CCInfo.getStackSize();
2522 if (isTailCall && !isSibCall) {
2529 assert(StackAlign &&
"data layout string is missing stack alignment");
2530 NumBytes =
alignTo(NumBytes, *StackAlign);
2535 SPDiff = NumReusableBytes - NumBytes;
2539 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (
unsigned)-SPDiff)
2555 RegsToPassVector RegsToPass;
2570 SDValue Src = OutVals[ArgIdx];
2573 if (!
Flags.isByVal())
2578 std::tie(Dst, DstInfo) =
2579 computeAddrForCallArg(dl, DAG, VA,
SDValue(),
true, SPDiff);
2580 ByValCopyKind
Copy = ByValNeedsCopyForTailCall(DAG, Src, Dst, Flags);
2582 if (Copy == NoCopy) {
2587 }
else if (Copy == CopyOnce) {
2591 ByValTemporaries[ArgIdx] = Src;
2593 assert(Copy == CopyViaTemp &&
"unexpected enum value");
2597 int TempFrameIdx = MFI.CreateStackObject(
2598 Flags.getByValSize(),
Flags.getNonZeroByValAlign(),
false);
2607 SDValue Ops[] = {Chain, Temp, Src, SizeNode, AlignNode};
2610 ByValTemporaries[ArgIdx] = Temp;
2613 if (!ByValCopyChains.
empty())
2623 bool AfterFormalArgLoads =
false;
2627 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
2629 ++i, ++realArgIdx) {
2631 SDValue Arg = OutVals[realArgIdx];
2633 bool isByVal =
Flags.isByVal();
2653 if (isTailCall && VA.
isMemLoc() && !AfterFormalArgLoads) {
2658 AfterFormalArgLoads =
true;
2670 auto ArgVT = Outs[realArgIdx].ArgVT;
2671 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2689 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2690 StackPtr, MemOpChains, isTailCall, SPDiff);
2694 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2695 StackPtr, MemOpChains, isTailCall, SPDiff);
2700 std::tie(DstAddr, DstInfo) =
2701 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2705 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2706 StackPtr, MemOpChains, isTailCall, SPDiff);
2708 if (realArgIdx == 0 &&
Flags.isReturned() && !
Flags.isSwiftSelf() &&
2709 Outs[0].VT == MVT::i32) {
2711 "unexpected calling convention register assignment");
2712 assert(!
Ins.empty() && Ins[0].VT == MVT::i32 &&
2713 "unexpected use of 'returned'");
2714 isThisReturn =
true;
2719 RegsToPass.push_back(std::make_pair(VA.
getLocReg(), Arg));
2720 }
else if (isByVal) {
2722 unsigned offset = 0;
2726 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2727 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2730 bool NeedsStackCopy;
2731 if (ByValTemporaries.
contains(realArgIdx)) {
2732 ByValSrc = ByValTemporaries[realArgIdx];
2733 NeedsStackCopy =
true;
2736 NeedsStackCopy = !isTailCall;
2740 if (CurByValIdx < ByValArgsCount) {
2741 unsigned RegBegin, RegEnd;
2742 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
2747 for (i = 0, j = RegBegin;
j < RegEnd; i++,
j++) {
2754 RegsToPass.push_back(std::make_pair(j, Load));
2759 offset = RegEnd - RegBegin;
2761 CCInfo.nextInRegsParam();
2766 if (NeedsStackCopy &&
Flags.getByValSize() > 4 * offset) {
2770 std::tie(Dst, DstInfo) =
2771 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2780 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2788 std::tie(DstAddr, DstInfo) =
2789 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2796 if (!MemOpChains.
empty())
2802 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
2803 Chain = DAG.
getCopyToReg(Chain, dl, RegsToPass[i].first,
2804 RegsToPass[i].second, InGlue);
2811 bool isDirect =
false;
2816 GVal =
G->getGlobal();
2817 bool isStub = !
TM.shouldAssumeDSOLocal(GVal) && Subtarget->
isTargetMachO();
2819 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->
isMClass());
2820 bool isLocalARMFunc =
false;
2823 if (Subtarget->genLongCalls()) {
2825 "long-calls codegen is not position independent!");
2829 if (isa<GlobalAddressSDNode>(Callee)) {
2830 if (Subtarget->genExecuteOnly()) {
2849 const char *
Sym = S->getSymbol();
2851 if (Subtarget->genExecuteOnly()) {
2870 }
else if (isa<GlobalAddressSDNode>(Callee)) {
2871 if (!PreferIndirect) {
2876 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !
ARMInterworking);
2878 if (isStub && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2890 "Windows is the only supported COFF target");
2894 else if (!
TM.shouldAssumeDSOLocal(GVal))
2910 const char *
Sym = S->getSymbol();
2911 if (isARMFunc && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2915 ARMPCLabelIndex, 4);
2929 assert(!isARMFunc && !isDirect &&
2930 "Cannot handle call to ARM function or direct call");
2933 "call to non-secure function would "
2934 "require passing arguments on stack",
2941 "call to non-secure function would return value through pointer",
2949 if (Subtarget->isThumb()) {
2952 else if (isCmseNSCall)
2954 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2959 if (!isDirect && !Subtarget->hasV5TOps())
2961 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2974 if (isTailCall && !isSibCall) {
2979 std::vector<SDValue> Ops;
2980 Ops.push_back(Chain);
2981 Ops.push_back(Callee);
2989 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2990 Ops.push_back(DAG.
getRegister(RegsToPass[i].first,
2991 RegsToPass[i].second.getValueType()));
3003 isThisReturn =
false;
3009 assert(Mask &&
"Missing call preserved mask for calling convention");
3013 Ops.push_back(InGlue);
3024 Chain = DAG.
getNode(CallOpc, dl, {MVT::Other, MVT::Glue}, Ops);
3036 Chain = DAG.
getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
3042 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
3043 InVals, isThisReturn,
3044 isThisReturn ? OutVals[0] :
SDValue(), isCmseNSCall);
3051void ARMTargetLowering::HandleByVal(
CCState *State,
unsigned &
Size,
3052 Align Alignment)
const {
3054 Alignment = std::max(Alignment,
Align(4));
3060 unsigned AlignInRegs = Alignment.
value() / 4;
3061 unsigned Waste = (ARM::R4 -
Reg) % AlignInRegs;
3062 for (
unsigned i = 0; i < Waste; ++i)
3068 unsigned Excess = 4 * (ARM::R4 -
Reg);
3075 if (NSAAOffset != 0 &&
Size > Excess) {
3087 unsigned ByValRegBegin =
Reg;
3088 unsigned ByValRegEnd = std::min<unsigned>(Reg +
Size / 4, ARM::R4);
3092 for (
unsigned i = Reg + 1; i != ByValRegEnd; ++i)
3098 Size = std::max<int>(
Size - Excess, 0);
3106bool ARMTargetLowering::IsEligibleForTailCallOptimization(
3132 if (!isa<GlobalAddressSDNode>(
Callee.getNode()) || isIndirect) {
3134 for (
Register R : {ARM::R0, ARM::R1, ARM::R2, ARM::R3})
3135 AddressRegisters.
insert(R);
3138 AddressRegisters.
insert(ARM::R12);
3141 AddressRegisters.
erase(
AL.getLocReg());
3142 if (AddressRegisters.
empty()) {
3143 LLVM_DEBUG(
dbgs() <<
"false (no reg to hold function pointer)\n");
3162 <<
" (guaranteed tail-call CC)\n");
3163 return CalleeCC == CallerCC;
3168 bool isCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
3170 if (isCalleeStructRet != isCallerStructRet) {
3186 (!
TT.isOSWindows() ||
TT.isOSBinFormatELF() ||
3187 TT.isOSBinFormatMachO())) {
3196 getEffectiveCallingConv(CalleeCC, isVarArg),
3197 getEffectiveCallingConv(CallerCC, CallerF.
isVarArg()), MF,
C, Ins,
3205 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
3206 if (CalleeCC != CallerCC) {
3207 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
3208 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) {
3227 LLVM_DEBUG(
dbgs() <<
"false (parameters in CSRs do not match)\n");
3246 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
3255 StringRef IntKind =
F.getFnAttribute(
"interrupt").getValueAsString();
3268 if (IntKind ==
"" || IntKind ==
"IRQ" || IntKind ==
"FIQ" ||
3271 else if (IntKind ==
"SWI" || IntKind ==
"UNDEF")
3275 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
3302 bool isLittleEndian = Subtarget->
isLittle();
3314 "secure entry function would return value through pointer",
3320 for (
unsigned i = 0, realRVLocIdx = 0;
3322 ++i, ++realRVLocIdx) {
3326 SDValue Arg = OutVals[realRVLocIdx];
3327 bool ReturnF16 =
false;
3362 auto RetVT = Outs[realRVLocIdx].ArgVT;
3384 DAG.
getVTList(MVT::i32, MVT::i32), Half);
3388 HalfGPRs.
getValue(isLittleEndian ? 0 : 1), Glue);
3394 HalfGPRs.
getValue(isLittleEndian ? 1 : 0), Glue);
3406 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
3408 fmrrd.
getValue(isLittleEndian ? 0 : 1), Glue);
3413 fmrrd.
getValue(isLittleEndian ? 1 : 0), Glue);
3457 return DAG.
getNode(RetNode, dl, MVT::Other, RetOps);
3460bool ARMTargetLowering::isUsedByReturnOnly(
SDNode *
N,
SDValue &Chain)
const {
3461 if (
N->getNumValues() != 1)
3463 if (!
N->hasNUsesOfValue(1, 0))
3471 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3473 TCChain =
Copy->getOperand(0);
3487 SDValue UseChain =
U->getOperand(0);
3495 if (
U->getOperand(
U->getNumOperands() - 1).getValueType() == MVT::Glue)
3503 if (!
Copy->hasOneUse())
3510 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3512 TCChain =
Copy->getOperand(0);
3517 bool HasRet =
false;
3532bool ARMTargetLowering::mayBeEmittedAsTailCall(
const CallInst *CI)
const {
3550 &&
"LowerWRITE_REGISTER called for non-i64 type argument.");
3566 EVT PtrVT =
Op.getValueType();
3576 if (Subtarget->genExecuteOnly()) {
3578 auto T =
const_cast<Type*
>(
CP->getType());
3579 auto C =
const_cast<Constant*
>(
CP->getConstVal());
3590 return LowerGlobalAddress(GA, DAG);
3595 Align CPAlign =
CP->getAlign();
3597 CPAlign = std::max(CPAlign,
Align(4));
3598 if (
CP->isMachineConstantPoolEntry())
3610 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3619 unsigned ARMPCLabelIndex = 0;
3622 const BlockAddress *BA = cast<BlockAddressSDNode>(
Op)->getBlockAddress();
3625 if (!IsPositionIndependent) {
3628 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3639 if (!IsPositionIndependent)
3670ARMTargetLowering::LowerGlobalTLSAddressDarwin(
SDValue Op,
3673 "This function expects a Darwin target");
3678 SDValue DescAddr = LowerGlobalAddressDarwin(
Op, DAG);
3684 MVT::i32,
DL, Chain, DescAddr,
3708 Chain, FuncTLVGet, DAG.
getRegister(ARM::R0, MVT::i32),
3714ARMTargetLowering::LowerGlobalTLSAddressWindows(
SDValue Op,
3731 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
3758 const auto *GA = cast<GlobalAddressSDNode>(
Op);
3775 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3797 Args.push_back(Entry);
3805 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
3806 return CallResult.first;
3828 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3836 PtrVT, dl, Chain,
Offset,
3838 Chain =
Offset.getValue(1);
3844 PtrVT, dl, Chain,
Offset,
3854 PtrVT, dl, Chain,
Offset,
3870 return LowerGlobalTLSAddressDarwin(
Op, DAG);
3873 return LowerGlobalTLSAddressWindows(
Op, DAG);
3882 return LowerToTLSGeneralDynamicModel(GA, DAG);
3885 return LowerToTLSExecModels(GA, DAG, model);
3894 while (!Worklist.
empty()) {
3896 if (isa<ConstantExpr>(U)) {
3901 auto *
I = dyn_cast<Instruction>(U);
3902 if (!
I ||
I->getParent()->getParent() !=
F)
3930 auto *GVar = dyn_cast<GlobalVariable>(GV);
3931 if (!GVar || !GVar->hasInitializer() ||
3932 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3933 !GVar->hasLocalLinkage())
3938 auto *
Init = GVar->getInitializer();
3940 Init->needsDynamicRelocation())
3949 auto *CDAInit = dyn_cast<ConstantDataArray>(
Init);
3952 unsigned RequiredPadding = 4 - (
Size % 4);
3953 bool PaddingPossible =
3954 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3959 unsigned PaddedSize =
Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3983 if (RequiredPadding != 4) {
3988 while (RequiredPadding--)
4000 ++NumConstpoolPromoted;
4005 if (
const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
4006 if (!(GV = GA->getAliaseeObject()))
4008 if (
const auto *V = dyn_cast<GlobalVariable>(GV))
4009 return V->isConstant();
4010 return isa<Function>(GV);
4018 return LowerGlobalAddressWindows(
Op, DAG);
4020 return LowerGlobalAddressELF(
Op, DAG);
4022 return LowerGlobalAddressDarwin(
Op, DAG);
4030 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4034 if (GV->
isDSOLocal() && !Subtarget->genExecuteOnly())
4047 }
else if (Subtarget->
isROPI() && IsRO) {
4052 }
else if (Subtarget->
isRWPI() && !IsRO) {
4077 if (Subtarget->
useMovt() || Subtarget->genExecuteOnly()) {
4096 "ROPI/RWPI not currently supported for Darwin");
4099 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4122 "Windows on ARM expects to use movw/movt");
4124 "ROPI/RWPI not currently supported for Windows");
4127 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4131 else if (!
TM.shouldAssumeDSOLocal(GV))
4155 DAG.
getVTList(MVT::i32, MVT::Other),
Op.getOperand(0),
4156 Op.getOperand(1), Val);
4173SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
4176 Op.getConstantOperandVal(
Op.getOperand(0).getValueType() == MVT::Other);
4180 case Intrinsic::arm_gnu_eabi_mcount: {
4189 assert(Mask &&
"Missing call preserved mask for calling convention");
4194 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
4198 if (Subtarget->isThumb())
4201 ARM::tBL_PUSHLR, dl, ResultTys,
4202 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
4203 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
4207 {ReturnAddress, Callee, RegisterMask, Chain}),
4216 unsigned IntNo =
Op.getConstantOperandVal(0);
4220 case Intrinsic::thread_pointer: {
4224 case Intrinsic::arm_cls: {
4225 const SDValue &Operand =
Op.getOperand(1);
4226 const EVT VTy =
Op.getValueType();
4237 case Intrinsic::arm_cls64: {
4240 const SDValue &Operand =
Op.getOperand(1);
4241 const EVT VTy =
Op.getValueType();
4264 case Intrinsic::eh_sjlj_lsda: {
4271 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
4281 if (IsPositionIndependent) {
4287 case Intrinsic::arm_neon_vabs:
4290 case Intrinsic::arm_neon_vabds:
4291 if (
Op.getValueType().isInteger())
4293 Op.getOperand(1),
Op.getOperand(2));
4295 case Intrinsic::arm_neon_vabdu:
4297 Op.getOperand(1),
Op.getOperand(2));
4298 case Intrinsic::arm_neon_vmulls:
4299 case Intrinsic::arm_neon_vmullu: {
4300 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
4303 Op.getOperand(1),
Op.getOperand(2));
4305 case Intrinsic::arm_neon_vminnm:
4306 case Intrinsic::arm_neon_vmaxnm: {
4307 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
4310 Op.getOperand(1),
Op.getOperand(2));
4312 case Intrinsic::arm_neon_vminu:
4313 case Intrinsic::arm_neon_vmaxu: {
4314 if (
Op.getValueType().isFloatingPoint())
4316 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
4319 Op.getOperand(1),
Op.getOperand(2));
4321 case Intrinsic::arm_neon_vmins:
4322 case Intrinsic::arm_neon_vmaxs: {
4324 if (!
Op.getValueType().isFloatingPoint()) {
4325 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4328 Op.getOperand(1),
Op.getOperand(2));
4330 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4333 Op.getOperand(1),
Op.getOperand(2));
4335 case Intrinsic::arm_neon_vtbl1:
4337 Op.getOperand(1),
Op.getOperand(2));
4338 case Intrinsic::arm_neon_vtbl2:
4340 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4341 case Intrinsic::arm_mve_pred_i2v:
4342 case Intrinsic::arm_mve_pred_v2i:
4345 case Intrinsic::arm_mve_vreinterpretq:
4348 case Intrinsic::arm_mve_lsll:
4350 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4351 case Intrinsic::arm_mve_asrl:
4353 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4364 if (!Subtarget->hasDataBarrier()) {
4368 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4369 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4380 }
else if (Subtarget->preferISHSTBarriers() &&
4389 DAG.
getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4397 (!Subtarget->
isThumb1Only() && Subtarget->hasV5TEOps())))
4399 return Op.getOperand(0);
4402 unsigned isRead =
~Op.getConstantOperandVal(2) & 1;
4404 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4406 return Op.getOperand(0);
4408 unsigned isData =
Op.getConstantOperandVal(4);
4409 if (Subtarget->isThumb()) {
4411 isRead = ~isRead & 1;
4412 isData = ~isData & 1;
4429 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
4430 return DAG.
getStore(
Op.getOperand(0), dl, FR,
Op.getOperand(1),
4438 const SDLoc &dl)
const {
4444 RC = &ARM::tGPRRegClass;
4446 RC = &ARM::GPRRegClass;
4460 MVT::i32, dl, Root, FIN,
4481 const Value *OrigArg,
4482 unsigned InRegsParamRecordIdx,
4483 int ArgOffset,
unsigned ArgSize)
const {
4498 unsigned RBegin, REnd;
4508 ArgOffset = -4 * (ARM::R4 - RBegin);
4518 for (
unsigned Reg = RBegin, i = 0;
Reg < REnd; ++
Reg, ++i) {
4527 if (!MemOps.
empty())
4536 unsigned TotalArgRegsSaveSize,
4537 bool ForceMutable)
const {
4548 CCInfo.
getStackSize(), std::max(4U, TotalArgRegsSaveSize));
4552bool ARMTargetLowering::splitValueIntoRegisterParts(
4554 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
4556 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4568SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4570 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
4571 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4584SDValue ARMTargetLowering::LowerFormalArguments(
4600 unsigned CurArgIdx = 0;
4612 unsigned ArgRegBegin = ARM::R4;
4619 if (!
Flags.isByVal())
4623 unsigned RBegin, REnd;
4625 ArgRegBegin = std::min(ArgRegBegin, RBegin);
4631 int lastInsIndex = -1;
4635 ArgRegBegin = std::min(ArgRegBegin, (
unsigned)
GPRArgRegs[RegIdx]);
4638 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4642 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4644 if (Ins[VA.
getValNo()].isOrigArg()) {
4645 std::advance(CurOrigArg,
4646 Ins[VA.
getValNo()].getOrigArgIndex() - CurArgIdx);
4658 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4665 MVT::f64, dl, Chain, FIN,
4668 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4676 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4680 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4681 RC = &ARM::HPRRegClass;
4682 else if (RegVT == MVT::f32)
4683 RC = &ARM::SPRRegClass;
4684 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4685 RegVT == MVT::v4bf16)
4686 RC = &ARM::DPRRegClass;
4687 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4688 RegVT == MVT::v8bf16)
4689 RC = &ARM::QPRRegClass;
4690 else if (RegVT == MVT::i32)
4692 : &ARM::GPRRegClass;
4738 assert(VA.
getValVT() != MVT::i64 &&
"i64 should already be lowered");
4744 if (index != lastInsIndex)
4752 if (
Flags.isByVal()) {
4753 assert(Ins[index].isOrigArg() &&
4754 "Byval arguments cannot be implicit");
4758 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4773 lastInsIndex = index;
4780 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.
getStackSize(),
4781 TotalArgRegsSaveSize);
4785 "secure entry function must not be variadic", dl.
getDebugLoc());
4796 assert(StackAlign &&
"data layout string is missing stack alignment");
4797 StackArgSize =
alignTo(StackArgSize, *StackAlign);
4806 "secure entry function requires arguments on stack", dl.
getDebugLoc());
4816 return CFP->getValueAPF().isPosZero();
4820 SDValue WrapperOp =
Op.getOperand(1).getOperand(0);
4822 if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
4823 return CFP->getValueAPF().isPosZero();
4826 Op->getValueType(0) == MVT::f64) {
4841 const SDLoc &dl)
const {
4843 unsigned C = RHSC->getZExtValue();
4909 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(RHS) &&
4911 unsigned Mask =
LHS.getConstantOperandVal(1);
4912 auto *RHSC = cast<ConstantSDNode>(
RHS.getNode());
4913 uint64_t RHSV = RHSC->getZExtValue();
4914 if (
isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4916 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4931 isa<ConstantSDNode>(RHS) &&
RHS->getAsZExtVal() == 0x80000000U &&
4933 LHS.getConstantOperandVal(1) < 31) {
4934 unsigned ShiftAmt =
LHS.getConstantOperandVal(1) + 1;
4977 bool Signaling)
const {
4978 assert(Subtarget->hasFP64() ||
RHS.getValueType() != MVT::f64);
4993std::pair<SDValue, SDValue>
4996 assert(
Op.getValueType() == MVT::i32 &&
"Unsupported value type");
5008 switch (
Op.getOpcode()) {
5021 DAG.
getVTList(
Op.getValueType(), MVT::i32), LHS, RHS)
5060 return std::make_pair(
Value, OverflowCmp);
5071 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Op, DAG, ARMcc);
5076 EVT VT =
Op.getValueType();
5119 EVT VT =
Op.getValueType();
5123 switch (
Op.getOpcode()) {
5148 EVT VT =
Op.getValueType();
5149 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->
isThumb1Only())
5159 switch (
Op->getOpcode()) {
5175 switch (
Op->getOpcode()) {
5194 DAG.
getNode(NewOpcode, dl, MVT::i32,
5205 unsigned Opc =
Cond.getOpcode();
5207 if (
Cond.getResNo() == 1 &&
5215 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5216 EVT VT =
Op.getValueType();
5218 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, OverflowCmp, DAG);
5228 dyn_cast<ConstantSDNode>(
Cond.getOperand(0));
5230 dyn_cast<ConstantSDNode>(
Cond.getOperand(1));
5232 if (CMOVTrue && CMOVFalse) {
5238 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5240 False = SelectFalse;
5241 }
else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5247 return getCMOV(dl,
Op.getValueType(), True, False,
Cond.getOperand(2),
5248 Cond.getOperand(3), DAG);
5263 bool &swpCmpOps,
bool &swpVselOps) {
5291 swpCmpOps = !swpCmpOps;
5292 swpVselOps = !swpVselOps;
5315 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5317 DAG.
getVTList(MVT::i32, MVT::i32), FalseVal);
5319 DAG.
getVTList(MVT::i32, MVT::i32), TrueVal);
5354 ((K ==
LHS && K == TrueVal) || (K ==
RHS && K == FalseVal))) ||
5356 ((K ==
RHS && K == TrueVal) || (K ==
LHS && K == FalseVal)));
5377 EVT VT =
Op.getValueType();
5384 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
5399 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5407 if (!isa<ConstantSDNode>(K1) || !isa<ConstantSDNode>(K2))
5410 int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue();
5411 int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue();
5412 int64_t PosVal = std::max(Val1, Val2);
5413 int64_t NegVal = std::min(Val1, Val2);
5459 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
5460 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5465 if (*K != KTmp || V != VTmp)
5476bool ARMTargetLowering::isUnsupportedFloatingType(
EVT VT)
const {
5480 return !Subtarget->hasFP64();
5482 return !Subtarget->hasFullFP16();
5487 EVT VT =
Op.getValueType();
5491 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->
isThumb2())
5503 if (VT == MVT::i32 &&
5523 if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
5524 LHS.getValueType() == MVT::i32 &&
RHS.getValueType() == MVT::i32) {
5527 unsigned Opcode = 0;
5529 if (TVal == ~FVal) {
5531 }
else if (TVal == ~FVal + 1) {
5533 }
else if (TVal + 1 == FVal) {
5535 }
else if (TVal == FVal + 1) {
5567 return DAG.
getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
5571 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5573 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5577 if (!
RHS.getNode()) {
5583 if (
LHS.getValueType() == MVT::i32) {
5595 TrueVal.getValueType() == MVT::f32 ||
5596 TrueVal.getValueType() == MVT::f64)) {
5610 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Cmp, DAG);
5622 (
TrueVal.getValueType() == MVT::f16 ||
5623 TrueVal.getValueType() == MVT::f32 ||
5624 TrueVal.getValueType() == MVT::f64)) {
5625 bool swpCmpOps =
false;
5626 bool swpVselOps =
false;
5640 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Cmp, DAG);
5643 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, Cmp, DAG);
5653 if (!
N->hasOneUse())
5656 if (!
N->getNumValues())
5658 EVT VT =
Op.getValueType();
5659 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5676 return DAG.
getLoad(MVT::i32,
SDLoc(
Op), Ld->getChain(), Ld->getBasePtr(),
5677 Ld->getPointerInfo(), Ld->getAlign(),
5678 Ld->getMemOperand()->getFlags());
5696 DAG.
getLoad(MVT::i32, dl, Ld->getChain(),
Ptr, Ld->getPointerInfo(),
5697 Ld->getAlign(), Ld->getMemOperand()->getFlags());
5699 EVT PtrType =
Ptr.getValueType();
5702 RetVal2 = DAG.
getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5703 Ld->getPointerInfo().getWithOffset(4),
5705 Ld->getMemOperand()->getFlags());
5723 bool LHSSeenZero =
false;
5725 bool RHSSeenZero =
false;
5727 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5738 if (
LHS.getValueType() == MVT::f32) {
5756 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5771 unsigned Opc =
Cond.getOpcode();
5774 if (
Cond.getResNo() == 1 &&
5784 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5807 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5809 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5813 if (!
RHS.getNode()) {
5821 unsigned Opc =
LHS.getOpcode();
5835 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
LHS.getValue(0), DAG, ARMcc);
5849 if (
LHS.getValueType() == MVT::i32) {
5858 if (
SDValue Result = OptimizeVFPBrcond(
Op, DAG))
5889 if (Subtarget->
isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5895 Addr,
Op.getOperand(2), JTI);
5901 Chain =
Addr.getValue(1);
5908 Chain =
Addr.getValue(1);
5914 EVT VT =
Op.getValueType();
5917 if (
Op.getValueType().getVectorElementType() == MVT::i32) {
5918 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
5926 const EVT OpTy =
Op.getOperand(0).getValueType();
5927 if (OpTy == MVT::v4f32)
5929 else if (OpTy == MVT::v4f16 && HasFullFP16)
5931 else if (OpTy == MVT::v8f16 && HasFullFP16)
5936 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5939 Op = DAG.
getNode(
Op.getOpcode(), dl, NewTy,
Op.getOperand(0));
5944 EVT VT =
Op.getValueType();
5948 bool IsStrict =
Op->isStrictFPOpcode();
5949 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
5951 if (isUnsupportedFloatingType(SrcVal.
getValueType())) {
5961 MakeLibCallOptions CallOptions;
5964 std::tie(Result, Chain) =
makeLibCall(DAG, LC,
Op.getValueType(), SrcVal,
5965 CallOptions, Loc, Chain);
5975 Loc,
Op.getValueType(), SrcVal);
5984 EVT VT =
Op.getValueType();
5985 EVT ToVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
5986 EVT FromVT =
Op.getOperand(0).getValueType();
5988 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5990 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5991 Subtarget->hasFP64())
5993 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5994 Subtarget->hasFullFP16())
5996 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5997 Subtarget->hasMVEFloatOps())
5999 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
6000 Subtarget->hasMVEFloatOps())
6003 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
6020 EVT VT =
Op.getValueType();
6023 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
6029 assert((
Op.getOperand(0).getValueType() == MVT::v4i16 ||
6030 Op.getOperand(0).getValueType() == MVT::v8i16) &&
6031 "Invalid type for custom lowering!");
6036 if (VT == MVT::v4f32)
6037 DestVecType = MVT::v4i32;
6038 else if (VT == MVT::v4f16 && HasFullFP16)
6039 DestVecType = MVT::v4i16;
6040 else if (VT == MVT::v8f16 && HasFullFP16)
6041 DestVecType = MVT::v8i16;
6047 switch (
Op.getOpcode()) {
6059 Op = DAG.
getNode(CastOpc, dl, DestVecType,
Op.getOperand(0));
6064 EVT VT =
Op.getValueType();
6067 if (isUnsupportedFloatingType(VT)) {
6075 MakeLibCallOptions CallOptions;
6077 CallOptions,
SDLoc(
Op)).first;
6088 EVT VT =
Op.getValueType();
6092 bool UseNEON = !InGPR && Subtarget->hasNEON();
6099 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
6106 if (SrcVT == MVT::f32) {
6112 }
else if (VT == MVT::f32)
6128 if (VT == MVT::f32) {
6140 if (SrcVT == MVT::f64)
6149 if (VT == MVT::f32) {
6173 EVT VT =
Op.getValueType();
6175 unsigned Depth =
Op.getConstantOperandVal(0);
6177 SDValue FrameAddr = LowerFRAMEADDR(
Op, DAG);
6196 EVT VT =
Op.getValueType();
6198 unsigned Depth =
Op.getConstantOperandVal(0);
6212 .
Case(
"sp", ARM::SP)
6227 assert(
N->getValueType(0) == MVT::i64
6228 &&
"ExpandREAD_REGISTER called for non-i64 type result.");
6231 DAG.
getVTList(MVT::i32, MVT::i32, MVT::Other),
6271 const APInt &APIntIndex = Index->getAPIntValue();
6273 NewIndex *= APIntIndex;
6303 EVT SrcVT =
Op.getValueType();
6304 EVT DstVT =
N->getValueType(0);
6306 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6307 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6311 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6312 (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
6313 if (Subtarget->hasFullFP16() && !Subtarget->hasBF16())
6320 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6324 if (SrcVT == MVT::i64 && TLI.
isTypeLegal(DstVT)) {
6336 if (DstVT == MVT::i64 && TLI.
isTypeLegal(SrcVT)) {
6373 EVT VT =
Op.getValueType();
6391 SDValue LoBigShift = DAG.
getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6399 ? DAG.
getNode(Opc, dl, VT, ShOpHi,
6416 EVT VT =
Op.getValueType();
6458 DAG.
getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6510 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6538 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6568 EVT VT =
N->getValueType(0);
6569 if (VT.
isVector() && ST->hasNEON()) {
6578 if (ElemTy == MVT::i8) {
6586 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6601 if (ElemTy == MVT::i64) {
6614 if (!ST->hasV6T2Ops())
6623 EVT VT =
N->getValueType(0);
6626 assert(ST->hasNEON() &&
"Custom ctpop lowering requires NEON.");
6627 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6628 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6629 "Unexpected type for custom ctpop lowering");
6637 unsigned EltSize = 8;
6660 Op =
Op.getOperand(0);
6662 APInt SplatBits, SplatUndef;
6663 unsigned SplatBitSize;
6666 !BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6668 SplatBitSize > ElementBits)
6679 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6683 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6694 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6699 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6700 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6709 EVT VT =
N->getValueType(0);
6731 "unexpected vector shift opcode");
6733 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
6734 unsigned VShiftOpc =
6736 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
6742 EVT ShiftVT =
N->getOperand(1).getValueType();
6745 unsigned VShiftOpc =
6747 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0), NegatedCount);
6752 EVT VT =
N->getValueType(0);
6761 "Unknown shift to lower!");
6763 unsigned ShOpc =
N->getOpcode();
6764 if (ST->hasMVEIntegerOps()) {
6794 DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6808 if (ST->isThumb1Only())
6813 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6829 bool Invert =
false;
6836 EVT VT =
Op.getValueType();
6844 assert(ST->hasMVEIntegerOps() &&
6845 "No hardware support for integer vector comparison!");
6847 if (
Op.getValueType().getVectorElementType() != MVT::i1)
6872 Merged = DAG.
getNOT(dl, Merged, CmpVT);
6882 switch (SetCCOpcode) {
6886 if (ST->hasMVEFloatOps()) {
6889 Invert =
true; [[fallthrough]];
6894 case ISD::SETLT: Swap =
true; [[fallthrough]];
6898 case ISD::SETLE: Swap =
true; [[fallthrough]];
6914 Result = DAG.
getNOT(dl, Result, VT);
6917 case ISD::SETUO: Invert =
true; [[fallthrough]];
6926 Result = DAG.
getNOT(dl, Result, VT);
6932 switch (SetCCOpcode) {
6935 if (ST->hasMVEIntegerOps()) {
6938 Invert =
true; [[fallthrough]];
6941 case ISD::SETLT: Swap =
true; [[fallthrough]];
6943 case ISD::SETLE: Swap =
true; [[fallthrough]];
6952 if (ST->hasNEON() && Opc ==
ARMCC::EQ) {
6968 Result = DAG.
getNOT(dl, Result, VT);
7002 Result = DAG.
getNOT(dl, Result, VT);
7014 assert(
LHS.getSimpleValueType().isInteger() &&
"SETCCCARRY is integer only.");
7041 unsigned OpCmode, Imm;
7052 switch (SplatBitSize) {
7057 assert((SplatBits & ~0xff) == 0 &&
"one byte splat value is too big");
7060 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
7065 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
7066 if ((SplatBits & ~0xff) == 0) {
7072 if ((SplatBits & ~0xff00) == 0) {
7075 Imm = SplatBits >> 8;
7085 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
7086 if ((SplatBits & ~0xff) == 0) {
7092 if ((SplatBits & ~0xff00) == 0) {
7095 Imm = SplatBits >> 8;
7098 if ((SplatBits & ~0xff0000) == 0) {
7101 Imm = SplatBits >> 16;
7104 if ((SplatBits & ~0xff000000) == 0) {
7107 Imm = SplatBits >> 24;
7114 if ((SplatBits & ~0xffff) == 0 &&
7115 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
7118 Imm = SplatBits >> 8;
7126 if ((SplatBits & ~0xffffff) == 0 &&
7127 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
7130 Imm = SplatBits >> 16;
7146 unsigned ImmMask = 1;
7148 for (
int ByteNum = 0; ByteNum < 8; ++ByteNum) {
7149 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
7151 }
else if ((SplatBits & BitMask) != 0) {
7160 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
7174 EVT VT =
Op.getValueType();
7175 bool IsDouble = (VT == MVT::f64);
7181 if (
ST->genExecuteOnly()) {
7183 assert((!
ST->isThumb1Only() ||
ST->hasV8MBaselineOps()) &&
7184 "Unexpected architecture");
7207 if (!
ST->hasVFP3Base())
7212 if (IsDouble && !Subtarget->hasFP64())
7219 if (IsDouble || !
ST->useNEONForSinglePrecisionFP()) {
7237 if (!
ST->hasNEON() || (!IsDouble && !
ST->useNEONForSinglePrecisionFP()))
7246 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7300 unsigned ExpectedElt = Imm;
7301 for (
unsigned i = 1; i < NumElts; ++i) {
7305 if (ExpectedElt == NumElts)
7308 if (M[i] < 0)
continue;
7309 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7317 bool &ReverseVEXT,
unsigned &Imm) {
7319 ReverseVEXT =
false;
7330 unsigned ExpectedElt = Imm;
7331 for (
unsigned i = 1; i < NumElts; ++i) {
7335 if (ExpectedElt == NumElts * 2) {
7340 if (M[i] < 0)
continue;
7341 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7356 return VT == MVT::v8i8 && M.size() == 8;
7361 if (Mask.size() == Elements * 2)
7362 return Index / Elements;
7363 return Mask[Index] == 0 ? 0 : 1;
7393 if (M.size() != NumElts && M.size() != NumElts*2)
7401 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7403 for (
unsigned j = 0; j < NumElts; j += 2) {
7404 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7405 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + NumElts + WhichResult))
7410 if (M.size() == NumElts*2)
7425 if (M.size() != NumElts && M.size() != NumElts*2)
7428 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7430 for (
unsigned j = 0; j < NumElts; j += 2) {
7431 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7432 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + WhichResult))
7437 if (M.size() == NumElts*2)
7457 if (M.size() != NumElts && M.size() != NumElts*2)
7460 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7462 for (
unsigned j = 0; j < NumElts; ++j) {
7463 if (M[i+j] >= 0 && (
unsigned) M[i+j] != 2 * j + WhichResult)
7468 if (M.size() == NumElts*2)
7487 if (M.size() != NumElts && M.size() != NumElts*2)
7490 unsigned Half = NumElts / 2;
7491 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7493 for (
unsigned j = 0; j < NumElts; j += Half) {
7494 unsigned Idx = WhichResult;
7495 for (
unsigned k = 0; k < Half; ++k) {
7496 int MIdx = M[i + j + k];
7497 if (MIdx >= 0 && (
unsigned) MIdx !=
Idx)
7504 if (M.size() == NumElts*2)
7528 if (M.size() != NumElts && M.size() != NumElts*2)
7531 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7533 unsigned Idx = WhichResult * NumElts / 2;
7534 for (
unsigned j = 0; j < NumElts; j += 2) {
7535 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7536 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx + NumElts))
7542 if (M.size() == NumElts*2)
7561 if (M.size() != NumElts && M.size() != NumElts*2)
7564 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7566 unsigned Idx = WhichResult * NumElts / 2;
7567 for (
unsigned j = 0; j < NumElts; j += 2) {
7568 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7569 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx))
7575 if (M.size() == NumElts*2)
7588 unsigned &WhichResult,
7591 if (
isVTRNMask(ShuffleMask, VT, WhichResult))
7593 if (
isVUZPMask(ShuffleMask, VT, WhichResult))
7595 if (
isVZIPMask(ShuffleMask, VT, WhichResult))
7613 if (NumElts != M.size())
7617 for (
unsigned i = 0; i != NumElts; ++i)
7618 if (M[i] >= 0 && M[i] != (
int) (NumElts - 1 - i))
7627 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7635 int Ofs = Top ? 1 : 0;
7636 int Upper = SingleSource ? 0 : NumElts;
7637 for (
int i = 0, e = NumElts / 2; i != e; ++i) {
7638 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7640 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs +
Upper)
7649 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7658 unsigned Offset = Top ? 0 : 1;
7659 unsigned N = SingleSource ? 0 : NumElts;
7660 for (
unsigned i = 0; i < NumElts; i += 2) {
7661 if (M[i] >= 0 && M[i] != (
int)i)
7663 if (M[i + 1] >= 0 && M[i + 1] != (
int)(
N + i +
Offset))
7672 if (NumElts != M.size())
7680 unsigned Off0 = rev ? NumElts / 2 : 0;
7681 unsigned Off1 = rev ? 0 : NumElts / 2;
7682 for (
unsigned i = 0; i < NumElts; i += 2) {
7683 if (M[i] >= 0 && M[i] != (
int)(Off0 + i / 2))
7685 if (M[i + 1] >= 0 && M[i + 1] != (
int)(Off1 + i / 2))
7701 if (!ST->hasMVEFloatOps())
7706 if (VT != MVT::v8f16)
7727 for (
unsigned i = 1; i < 4; i++) {
7754 if (!ST->hasMVEFloatOps())
7759 if (VT != MVT::v4f32)
7775 for (
unsigned i = 1; i < 4; i++) {
7796 if (!isa<ConstantSDNode>(
N))
7798 Val =
N->getAsZExtVal();
7800 if (ST->isThumb1Only()) {
7801 if (Val <= 255 || ~Val <= 255)
7813 EVT VT =
Op.getValueType();
7815 assert(ST->hasMVEIntegerOps() &&
"LowerBUILD_VECTOR_i1 called without MVE!");
7819 unsigned BitsPerBool;
7823 }
else if (NumElts == 4) {
7826 }
else if (NumElts == 8) {
7829 }
else if (NumElts == 16) {
7838 if (!isa<ConstantSDNode>(FirstOp) &&
7840 return U.get().isUndef() || U.get() == FirstOp;
7848 unsigned Bits32 = 0;
7849 for (
unsigned i = 0; i < NumElts; ++i) {
7851 if (!isa<ConstantSDNode>(V) && !V.isUndef())
7853 bool BitSet = V.isUndef() ?
false : V->getAsZExtVal();
7855 Bits32 |= BoolMask << (i * BitsPerBool);
7861 for (
unsigned i = 0; i < NumElts; ++i) {
7863 if (isa<ConstantSDNode>(V) || V.isUndef())
7874 if (!ST->hasMVEIntegerOps())
7878 EVT VT =
Op.getValueType();
7888 if (
N != 1 &&
N != 2 &&
N != 4 &&
N != 8)
7892 for (
unsigned I = 2;
I < NumElts;
I++) {
7908 switch (
N->getOpcode()) {
7919 return N->getOperand(1).getNode() ==
Op;
7921 switch (
N->getConstantOperandVal(0)) {
7922 case Intrinsic::arm_mve_add_predicated:
7923 case Intrinsic::arm_mve_mul_predicated:
7924 case Intrinsic::arm_mve_qadd_predicated:
7925 case Intrinsic::arm_mve_vhadd:
7926 case Intrinsic::arm_mve_hadd_predicated:
7927 case Intrinsic::arm_mve_vqdmulh:
7928 case Intrinsic::arm_mve_qdmulh_predicated:
7929 case Intrinsic::arm_mve_vqrdmulh:
7930 case Intrinsic::arm_mve_qrdmulh_predicated:
7931 case Intrinsic::arm_mve_vqdmull:
7932 case Intrinsic::arm_mve_vqdmull_predicated:
7934 case Intrinsic::arm_mve_sub_predicated:
7935 case Intrinsic::arm_mve_qsub_predicated:
7936 case Intrinsic::arm_mve_vhsub:
7937 case Intrinsic::arm_mve_hsub_predicated:
7938 return N->getOperand(2).getNode() ==
Op;
7953 EVT VT =
Op.getValueType();
7961 APInt SplatBits, SplatUndef;
7962 unsigned SplatBitSize;
7964 if (BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7971 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7973 [BVN](
const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
7974 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7975 : SplatBitSize == 16 ? MVT::v8i16
7982 if ((
ST->hasNEON() && SplatBitSize <= 64) ||
7983 (
ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7988 SplatBitSize, DAG, dl, VmovVT, VT,
VMOVModImm);
7996 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7998 NegatedImm, SplatUndef.
getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
8006 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
8016 if (
ST->hasMVEIntegerOps() &&
8017 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
8018 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
8019 : SplatBitSize == 16 ? MVT::v8i16
8036 bool isOnlyLowElement =
true;
8037 bool usesOnlyOneValue =
true;
8038 bool hasDominantValue =
false;
8045 for (
unsigned i = 0; i < NumElts; ++i) {
8050 isOnlyLowElement =
false;
8051 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
8054 unsigned &Count = ValueCounts[
V];
8057 if (++Count > (NumElts / 2)) {
8058 hasDominantValue =
true;
8062 if (ValueCounts.
size() != 1)
8063 usesOnlyOneValue =
false;
8064 if (!
Value.getNode() && !ValueCounts.
empty())
8067 if (ValueCounts.
empty())
8079 if (hasDominantValue && EltSize <= 32) {
8090 (constIndex = dyn_cast<ConstantSDNode>(
Value->getOperand(1)))) {
8095 if (VT !=
Value->getOperand(0).getValueType()) {
8108 if (!usesOnlyOneValue) {
8111 for (
unsigned I = 0;
I < NumElts; ++
I) {
8126 assert(FVT == MVT::f32 || FVT == MVT::f16);
8127 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
8128 for (
unsigned i = 0; i < NumElts; ++i)
8133 Val = LowerBUILD_VECTOR(Val, DAG, ST);
8137 if (usesOnlyOneValue) {
8164 if (
ST->hasNEON() && VT.
is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
8184 if (EltSize >= 32) {
8190 for (
unsigned i = 0; i < NumElts; ++i)
8204 for (
unsigned i = 0 ; i < NumElts; ++i) {
8223 EVT VT =
Op.getValueType();
8226 struct ShuffleSourceInfo {
8228 unsigned MinElt = std::numeric_limits<unsigned>::max();
8229 unsigned MaxElt = 0;
8239 int WindowScale = 1;
8241 ShuffleSourceInfo(
SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
8243 bool operator ==(
SDValue OtherVec) {
return Vec == OtherVec; }
8249 for (
unsigned i = 0; i < NumElts; ++i) {
8257 }
else if (!isa<ConstantSDNode>(
V.getOperand(1))) {
8264 SDValue SourceVec =
V.getOperand(0);
8266 if (Source == Sources.
end())
8270 unsigned EltNo =
V.getConstantOperandVal(1);
8277 if (Sources.
size() > 2)
8283 for (
auto &Source : Sources) {
8284 EVT SrcEltTy =
Source.Vec.getValueType().getVectorElementType();
8285 if (SrcEltTy.
bitsLT(SmallestEltTy))
8286 SmallestEltTy = SrcEltTy;
8288 unsigned ResMultiplier =
8296 for (
auto &Src : Sources) {
8297 EVT SrcVT = Src.ShuffleVec.getValueType();
8301 if (SrcVTSize == VTSize)
8310 if (SrcVTSize < VTSize) {
8311 if (2 * SrcVTSize != VTSize)
8317 DAG.
getUNDEF(Src.ShuffleVec.getValueType()));
8321 if (SrcVTSize != 2 * VTSize)
8324 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8329 if (Src.MinElt >= NumSrcElts) {
8334 Src.WindowBase = -NumSrcElts;
8335 }
else if (Src.MaxElt < NumSrcElts) {
8352 Src.WindowBase = -Src.MinElt;
8359 for (
auto &Src : Sources) {
8360 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8361 if (SrcEltTy == SmallestEltTy)
8366 Src.WindowBase *= Src.WindowScale;
8371 for (
auto Src : Sources)
8372 assert(Src.ShuffleVec.getValueType() == ShuffleVT);
8380 if (
Entry.isUndef())
8384 int EltNo = cast<ConstantSDNode>(
Entry.getOperand(1))->getSExtValue();
8389 EVT OrigEltTy =
Entry.getOperand(0).getValueType().getVectorElementType();
8392 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8396 int *LaneMask = &
Mask[i * ResMultiplier];
8398 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8399 ExtractBase += NumElts * (Src - Sources.begin());
8400 for (
int j = 0;
j < LanesDefined; ++
j)
8401 LaneMask[j] = ExtractBase + j;
8407 assert(Sources.size() <= 2 &&
"Too many sources!");
8410 for (
unsigned i = 0; i < Sources.size(); ++i)
8439 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8459 unsigned PFIndexes[4];
8460 for (
unsigned i = 0; i != 4; ++i) {
8464 PFIndexes[i] = M[i];
8468 unsigned PFTableIndex =
8469 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8471 unsigned Cost = (PFEntry >> 30);
8477 bool ReverseVEXT, isV_UNDEF;
8478 unsigned Imm, WhichResult;
8481 if (EltSize >= 32 ||
8488 else if (Subtarget->hasNEON() &&
8493 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8496 else if (Subtarget->hasMVEIntegerOps() &&
8500 else if (Subtarget->hasMVEIntegerOps() &&
8514 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8515 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8516 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8519 if (LHSID == (1*9+2)*9+3)
return LHS;
8520 assert(LHSID == ((4*9+5)*9+6)*9+7 &&
"Illegal OP_COPY!");
8577 for (
int I : ShuffleMask)
8580 if (V2.getNode()->isUndef())
8590 EVT VT =
Op.getValueType();
8592 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8593 "Expect an v8i16/v16i8 type");
8599 std::vector<int> NewMask;
8603 NewMask.push_back(i);
8643 if (VT != MVT::v16i1)
8659 EVT VT =
Op.getValueType();
8663 assert(ST->hasMVEIntegerOps() &&
8664 "No support for vector shuffle of boolean predicates");
8690 "Expected identical vector type in expanded i1 shuffle!");
8694 PredAsVector2, ShuffleMask);
8699 if (VT == MVT::v2i1) {
8716 EVT VT =
Op.getValueType();
8720 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8721 "Unexpected vector type");
8723 int QuarterSize = NumElts / 4;
8732 for (
int i = 0; i <
Length; i++) {
8733 if (ShuffleMask[Start + i] >= 0) {
8734 if (ShuffleMask[Start + i] %
Length != i)
8736 MovIdx = ShuffleMask[Start + i] /
Length;
8744 for (
int i = 1; i <
Length; i++) {
8745 if (ShuffleMask[Start + i] >= 0 &&
8746 (ShuffleMask[Start + i] /
Length != MovIdx ||
8747 ShuffleMask[Start + i] %
Length != i))
8753 for (
int Part = 0; Part < 4; ++Part) {
8755 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8759 Input =
Op->getOperand(1);
8769 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8774 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8776 for (
int Part = 0; Part < 4; ++Part)
8777 for (
int i = 0; i < QuarterSize; i++)
8779 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8781 VT, dl,
Op->getOperand(0),
Op->getOperand(1), NewShuffleMask);
8784 for (
int Part = 0; Part < 4; ++Part)
8800 EVT VT =
Op.getValueType();
8812 for (
int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8816 if (Mask[i] != i + BaseOffset) {
8817 if (OffElement == -1)
8823 return NonUndef > 2 && OffElement != -1;
8827 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8829 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8840 ShuffleMask[OffElement] < (
int)NumElts ? V1 : V2,
8851 EVT VT =
Op.getValueType();
8855 if (ST->hasMVEIntegerOps() && EltSize == 1)
8866 if (EltSize <= 32) {
8870 if (Lane == -1) Lane = 0;
8881 bool IsScalarToVector =
true;
8884 IsScalarToVector =
false;
8887 if (IsScalarToVector)
8894 bool ReverseVEXT =
false;
8896 if (ST->hasNEON() &&
isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
8920 unsigned WhichResult = 0;
8921 bool isV_UNDEF =
false;
8922 if (ST->hasNEON()) {
8924 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8931 if (ST->hasMVEIntegerOps()) {
8966 }) &&
"Unexpected shuffle index into UNDEF operand!");
8969 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
8972 assert((WhichResult == 0) &&
8973 "In-place shuffle of concat can only have one result!");
8982 if (ST->hasMVEIntegerOps() && EltSize <= 32) {
8986 for (
bool Top : {
false,
true}) {
8987 for (
bool SingleSource : {
false,
true}) {
8988 if (
isTruncMask(ShuffleMask, VT, Top, SingleSource)) {
8993 SingleSource ? V1 : V2);
9009 unsigned PFIndexes[4];
9010 for (
unsigned i = 0; i != 4; ++i) {
9011 if (ShuffleMask[i] < 0)
9014 PFIndexes[i] = ShuffleMask[i];
9018 unsigned PFTableIndex =
9019 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
9021 unsigned Cost = (PFEntry >> 30);
9027 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
9028 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
9038 if (EltSize >= 32) {
9046 for (
unsigned i = 0; i < NumElts; ++i) {
9047 if (ShuffleMask[i] < 0)
9051 ShuffleMask[i] < (
int)NumElts ? V1 : V2,
9059 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
9063 if (ST->hasNEON() && VT == MVT::v8i8)
9067 if (ST->hasMVEIntegerOps())
9076 EVT VecVT =
Op.getOperand(0).getValueType();
9079 assert(ST->hasMVEIntegerOps() &&
9080 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9084 unsigned Lane =
Op.getConstantOperandVal(2);
9085 unsigned LaneWidth =
9087 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
9099 if (!isa<ConstantSDNode>(Lane))
9105 if (Subtarget->hasMVEIntegerOps() &&
9106 Op.getValueType().getScalarSizeInBits() == 1)
9130 IVecIn, IElt, Lane);
9139 EVT VecVT =
Op.getOperand(0).getValueType();
9142 assert(ST->hasMVEIntegerOps() &&
9143 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9147 unsigned Lane =
Op.getConstantOperandVal(1);
9148 unsigned LaneWidth =
9159 if (!isa<ConstantSDNode>(Lane))
9179 assert(
Op.getValueType().getScalarSizeInBits() == 1 &&
9180 "Unexpected custom CONCAT_VECTORS lowering");
9182 "Unexpected custom CONCAT_VECTORS lowering");
9183 assert(ST->hasMVEIntegerOps() &&
9184 "CONCAT_VECTORS lowering only supported for MVE");
9188 EVT Op2VT = V2.getValueType();
9189 assert(Op1VT == Op2VT &&
"Operand types don't match!");
9190 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
9191 "Unexpected i1 concat operations!");
9204 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
9218 auto ExtractInto = [&DAG, &dl](
SDValue NewV,
SDValue ConVec,
unsigned &j) {
9219 EVT NewVT = NewV.getValueType();
9220 EVT ConcatVT = ConVec.getValueType();
9221 unsigned ExtScale = 1;
9222 if (NewVT == MVT::v2f64) {
9236 ConVec = ExtractInto(NewV1, ConVec, j);
9237 ConVec = ExtractInto(NewV2, ConVec, j);
9247 while (ConcatOps.
size() > 1) {
9248 for (
unsigned I = 0, E = ConcatOps.
size();
I != E;
I += 2) {
9251 ConcatOps[
I / 2] = ConcatPair(V1, V2);
9255 return ConcatOps[0];
9260 EVT VT =
Op->getValueType(0);
9267 "unexpected CONCAT_VECTORS");
9288 EVT VT =
Op.getValueType();
9291 unsigned Index = V2->getAsZExtVal();
9294 "Unexpected custom EXTRACT_SUBVECTOR lowering");
9295 assert(ST->hasMVEIntegerOps() &&
9296 "EXTRACT_SUBVECTOR lowering only supported for MVE");
9306 EVT SubVT = MVT::v4i32;
9308 for (
unsigned i = Index, j = 0; i < (Index + NumElts); i++, j += 2) {
9323 for (
unsigned i = Index, j = 0; i < (Index + NumElts); i++, j++) {
9339 assert(ST->hasMVEIntegerOps() &&
"Expected MVE!");
9340 EVT VT =
N->getValueType(0);
9341 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9342 "Expected a vector i1 type!");
9344 EVT FromVT =
Op.getValueType();
9355 if (!Subtarget->hasMVEIntegerOps())
9358 EVT ToVT =
N->getValueType(0);
9401 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9403 EVT FromVT =
N->getOperand(0).getValueType();
9404 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9415 if (!Subtarget->hasMVEIntegerOps())
9420 EVT ToVT =
N->getValueType(0);
9421 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9424 EVT FromVT =
Op.getValueType();
9425 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9436 SDValue Ext1 = Ext.getValue(1);
9439 Ext = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext);
9440 Ext1 = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext1);
9452 EVT VT =
N->getValueType(0);
9454 SDNode *BVN =
N->getOperand(0).getNode();
9459 unsigned HiElt = 1 - LoElt;
9464 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9480 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
9481 SDNode *Elt =
N->getOperand(i).getNode();
9484 unsigned HalfSize = EltSize / 2;
9486 if (!
isIntN(HalfSize,
C->getSExtValue()))
9489 if (!
isUIntN(HalfSize,
C->getZExtValue()))
9528 switch (OrigSimpleTy) {
9544 unsigned ExtOpcode) {
9567 if (ExtendedTy == LD->getMemoryVT())
9568 return DAG.
getLoad(LD->getMemoryVT(),
SDLoc(LD), LD->getChain(),
9569 LD->getBasePtr(), LD->getPointerInfo(), LD->getAlign(),
9570 LD->getMemOperand()->getFlags());
9576 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
9577 LD->getMemoryVT(), LD->getAlign(),
9578 LD->getMemOperand()->getFlags());
9591 N->getOperand(0)->getValueType(0),
9597 "Expected extending load");
9603 DAG.
getNode(Opcode,
SDLoc(newLoad), LD->getValueType(0), newLoad);
9612 SDNode *BVN =
N->getOperand(0).getNode();
9614 BVN->
getValueType(0) == MVT::v4i32 &&
"expected v4i32 BUILD_VECTOR");
9622 EVT VT =
N->getValueType(0);
9628 for (
unsigned i = 0; i != NumElts; ++i) {
9629 const APInt &CInt =
N->getConstantOperandAPInt(i);
9638 unsigned Opcode =
N->getOpcode();
9640 SDNode *N0 =
N->getOperand(0).getNode();
9641 SDNode *N1 =
N->getOperand(1).getNode();
9649 unsigned Opcode =
N->getOpcode();
9651 SDNode *N0 =
N->getOperand(0).getNode();
9652 SDNode *N1 =
N->getOperand(1).getNode();
9662 EVT VT =
Op.getValueType();
9664 "unexpected type for custom-lowering ISD::MUL");
9665 SDNode *N0 =
Op.getOperand(0).getNode();
9666 SDNode *N1 =
Op.getOperand(1).getNode();
9667 unsigned NewOpc = 0;
9671 if (isN0SExt && isN1SExt)
9676 if (isN0ZExt && isN1ZExt)
9678 else if (isN1SExt || isN1ZExt) {
9695 if (VT == MVT::v2i64)
9712 "unexpected types for extended operands to VMULL");
9713 return DAG.
getNode(NewOpc,
DL, VT, Op0, Op1);
9748 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9782 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9785 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9806 EVT VT =
Op.getValueType();
9807 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9808 "unexpected type for custom-lowering ISD::SDIV");
9815 if (VT == MVT::v8i8) {
9843 EVT VT =
Op.getValueType();
9844 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9845 "unexpected type for custom-lowering ISD::UDIV");
9852 if (VT == MVT::v8i8) {
9891 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9894 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9898 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9919 EVT VT =
N->getValueType(0);
9933 Op.getOperand(1), Carry);
9947 Op.getOperand(1), Carry);
9980 bool ShouldUseSRet = Subtarget->
isAPCS_ABI();
9982 if (ShouldUseSRet) {
9992 Entry.IsSExt =
false;
9993 Entry.IsZExt =
false;
9994 Entry.IsSRet =
true;
9995 Args.push_back(Entry);
10002 Entry.IsSExt =
false;
10003 Entry.IsZExt =
false;
10004 Args.push_back(Entry);
10007 (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
10017 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
10019 if (!ShouldUseSRet)
10020 return CallResult.first;
10039 EVT VT =
Op.getValueType();
10040 assert((VT == MVT::i32 || VT == MVT::i64) &&
10041 "unexpected type for custom lowering DIV");
10047 const char *
Name =
nullptr;
10049 Name = (VT == MVT::i32) ?
"__rt_sdiv" :
"__rt_sdiv64";
10051 Name = (VT == MVT::i32) ?
"__rt_udiv" :
"__rt_udiv64";
10057 for (
auto AI : {1, 0}) {
10059 Arg.Node =
Op.getOperand(AI);
10060 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.
getContext());
10061 Args.push_back(Arg);
10064 CallLoweringInfo CLI(DAG);
10068 ES, std::move(Args));
10078ARMTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
10086 const bool MinSize =
ST.hasMinSize();
10087 const bool HasDivide =
ST.isThumb() ?
ST.hasDivideInThumbMode()
10088 :
ST.hasDivideInARMMode();
10092 if (
N->getOperand(0).getValueType().isVector())
10097 if (!(MinSize && HasDivide))
10110 if (Divisor.
sgt(128))
10118 assert(
Op.getValueType() == MVT::i32 &&
10119 "unexpected type for custom lowering DIV");
10125 return LowerWindowsDIVLibCall(
Op, DAG,
Signed, DBZCHK);
10131 if (
N->getValueType(0) == MVT::i32)
10139void ARMTargetLowering::ExpandDIV_Windows(
10145 assert(
Op.getValueType() == MVT::i64 &&
10146 "unexpected type for custom lowering DIV");
10163 EVT MemVT = LD->getMemoryVT();
10164 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10165 MemVT == MVT::v16i1) &&
10166 "Expected a predicate type!");
10167 assert(MemVT ==
Op.getValueType());
10169 "Expected a non-extending load");
10170 assert(LD->isUnindexed() &&
"Expected a unindexed load");
10184 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(),
10186 LD->getMemOperand());
10193 if (MemVT != MVT::v16i1)
10202 EVT MemVT =
LD->getMemoryVT();
10203 assert(
LD->isUnindexed() &&
"Loads should be unindexed at this point.");
10205 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10211 {LD->getChain(), LD->getBasePtr()}, MemVT,
LD->getMemOperand());
10221 EVT MemVT = ST->getMemoryVT();
10222 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10223 MemVT == MVT::v16i1) &&
10224 "Expected a predicate type!");
10225 assert(MemVT == ST->getValue().getValueType());
10226 assert(!ST->isTruncatingStore() &&
"Expected a non-extending store");
10227 assert(ST->isUnindexed() &&
"Expected a unindexed store");
10232 SDValue Build = ST->getValue();
10233 if (MemVT != MVT::v16i1) {
10252 ST->getChain(), dl, GRP, ST->getBasePtr(),
10254 ST->getMemOperand());
10260 EVT MemVT = ST->getMemoryVT();
10261 assert(ST->isUnindexed() &&
"Stores should be unindexed at this point.");
10263 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10279 {ST->getChain(), Lo, Hi, ST->getBasePtr()},
10280 MemVT, ST->getMemOperand());
10281 }
else if (Subtarget->hasMVEIntegerOps() &&
10282 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10283 MemVT == MVT::v16i1))) {
10298 MVT VT =
Op.getSimpleValueType();
10300 SDValue PassThru =
N->getPassThru();
10311 VT, dl,
N->getChain(),
N->getBasePtr(),
N->getOffset(), Mask, ZeroVec,
10312 N->getMemoryVT(),
N->getMemOperand(),
N->getAddressingMode(),
10313 N->getExtensionType(),
N->isExpandingLoad());
10318 if (!PassThru.
isUndef() && !PassThruIsCastZero)
10325 if (!ST->hasMVEIntegerOps())
10329 unsigned BaseOpcode = 0;
10330 switch (
Op->getOpcode()) {
10346 unsigned NumActiveLanes = NumElts;
10348 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10349 NumActiveLanes == 2) &&
10350 "Only expected a power 2 vector size");
10354 while (NumActiveLanes > 4) {
10357 Op0 = DAG.
getNode(BaseOpcode, dl, VT, Op0, Rev);
10358 NumActiveLanes /= 2;
10362 if (NumActiveLanes == 4) {
10372 SDValue Res0 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10373 SDValue Res1 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext2, Ext3,
Op->getFlags());
10374 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res0, Res1,
Op->getFlags());
10380 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10384 if (EltVT !=
Op->getValueType(0))
10391 if (!ST->hasMVEFloatOps())
10398 if (!ST->hasNEON())
10406 unsigned PairwiseIntrinsic = 0;
10407 switch (
Op->getOpcode()) {
10411 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
10414 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
10417 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
10420 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10426 unsigned NumActiveLanes = NumElts;
10428 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10429 NumActiveLanes == 2) &&
10430 "Only expected a power 2 vector size");
10436 VT =
Lo.getValueType();
10438 NumActiveLanes /= 2;
10442 while (NumActiveLanes > 1) {
10444 NumActiveLanes /= 2;
10451 if (EltVT !=
Op.getValueType()) {
10452 unsigned Extend = 0;
10453 switch (
Op->getOpcode()) {
10465 Res = DAG.
getNode(Extend, dl,
Op.getValueType(), Res);
10487 SDValue Ops[] = {
N->getOperand(0),
10497 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
10510 const SDValue Ops[] = {RegClass, V0, SubReg0, V1, SubReg1};
10512 DAG.
getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
10516 SDLoc dl(V.getNode());
10517 auto [VLo, VHi] = DAG.
SplitScalar(V, dl, MVT::i32, MVT::i32);
10527 assert(
N->getValueType(0) == MVT::i64 &&
10528 "AtomicCmpSwap on types less than 64 should be legal");
10537 ARM::CMP_SWAP_64,
SDLoc(
N),
10538 DAG.
getVTList(MVT::Untyped, MVT::Untyped, MVT::Other), Ops);
10557 EVT VT =
Op.getValueType();
10566 if (isUnsupportedFloatingType(
LHS.getValueType())) {
10568 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS, Chain, IsSignaling);
10569 if (!
RHS.getNode()) {
10584 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10585 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, Cmp, DAG);
10587 ARMcc = DAG.
getConstant(CondCode2, dl, MVT::i32);
10588 Result = getCMOV(dl, VT, Result, True, ARMcc, Cmp, DAG);
10605 MakeLibCallOptions CallOptions;
10606 MVT SVT =
Op.getOperand(0).getSimpleValueType();
10609 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
10615 switch (
Op.getOpcode()) {
10647 case ISD::BITCAST:
return ExpandBITCAST(
Op.getNode(), DAG, Subtarget);
10651 case ISD::SREM:
return LowerREM(
Op.getNode(), DAG);
10652 case ISD::UREM:
return LowerREM(
Op.getNode(), DAG);
10674 return LowerSET_FPMODE(
Op, DAG);
10676 return LowerRESET_FPMODE(
Op, DAG);
10680 return LowerDIV_Windows(
Op, DAG,
true);
10684 return LowerDIV_Windows(
Op, DAG,
false);
10691 return LowerSignedALUO(
Op, DAG);
10694 return LowerUnsignedALUO(
Op, DAG);
10728 return LowerDYNAMIC_STACKALLOC(
Op, DAG);
10737 return LowerSPONENTRY(
Op, DAG);
10739 return LowerFP_TO_BF16(
Op, DAG);
10746 unsigned IntNo =
N->getConstantOperandVal(0);
10748 if (IntNo == Intrinsic::arm_smlald)
10750 else if (IntNo == Intrinsic::arm_smlaldx)
10752 else if (IntNo == Intrinsic::arm_smlsld)
10754 else if (IntNo == Intrinsic::arm_smlsldx)
10761 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(3), dl, MVT::i32, MVT::i32);
10765 N->getOperand(1),
N->getOperand(2),
10777 switch (
N->getOpcode()) {
10784 Res = ExpandBITCAST(
N, DAG, Subtarget);
10793 Res = LowerREM(
N, DAG);
10797 Res = LowerDivRem(
SDValue(
N, 0), DAG);
10851 "ROPI/RWPI not currently supported with SjLj");
10860 bool isThumb = Subtarget->isThumb();
10861 bool isThumb2 = Subtarget->
isThumb2();
10864 unsigned PCAdj = (
isThumb || isThumb2) ? 4 : 8;
10870 : &ARM::GPRRegClass;
10888 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10894 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10900 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10918 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10923 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10928 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10933 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
10939 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
10954 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10960 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10984 : &ARM::GPRnopcRegClass;
10989 unsigned MaxCSNum = 0;
10997 if (!
II.isEHLabel())
11001 if (!MF->hasCallSiteLandingPad(
Sym))
continue;
11004 for (
unsigned Idx : CallSiteIdxs) {
11005 CallSiteNumToLPad[
Idx].push_back(&BB);
11006 MaxCSNum = std::max(MaxCSNum,
Idx);
11013 std::vector<MachineBasicBlock*> LPadList;
11016 for (
unsigned I = 1;
I <= MaxCSNum; ++
I) {
11019 LPadList.push_back(
MBB);
11024 assert(!LPadList.empty() &&
11025 "No landing pad destinations for the dispatch jump table!");
11039 unsigned trap_opcode;
11040 if (Subtarget->isThumb())
11041 trap_opcode = ARM::tTRAP;
11043 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP;
11052 MF->insert(MF->end(), DispatchBB);
11053 MF->insert(MF->end(), DispContBB);
11054 MF->insert(MF->end(), TrapBB);
11058 SetupEntryBlockForSjLj(
MI,
MBB, DispatchBB, FI);
11065 MIB =
BuildMI(DispatchBB, dl,
TII->get(ARM::Int_eh_sjlj_dispatchsetup));
11077 unsigned NumLPads = LPadList.size();
11079 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11080 BuildMI(DispatchBB, dl,
TII->get(ARM::t2LDRi12), NewVReg1)
11086 if (NumLPads < 256) {
11087 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPri))
11089 .
addImm(LPadList.size())
11092 Register VReg1 =
MRI->createVirtualRegister(TRC);
11093 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVi16), VReg1)
11094 .
addImm(NumLPads & 0xFFFF)
11097 unsigned VReg2 = VReg1;
11098 if ((NumLPads & 0xFFFF0000) != 0) {
11099 VReg2 =
MRI->createVirtualRegister(TRC);
11100 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVTi16), VReg2)
11106 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPrr))
11112 BuildMI(DispatchBB, dl,
TII->get(ARM::t2Bcc))
11117 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11118 BuildMI(DispContBB, dl,
TII->get(ARM::t2LEApcrelJT), NewVReg3)
11122 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11123 BuildMI(DispContBB, dl,
TII->get(ARM::t2ADDrs), NewVReg4)
11130 BuildMI(DispContBB, dl,
TII->get(ARM::t2BR_JT))
11134 }
else if (Subtarget->isThumb()) {
11135 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11136 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRspi), NewVReg1)
11142 if (NumLPads < 256) {
11143 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPi8))
11150 const Constant *
C = ConstantInt::get(Int32Ty, NumLPads);
11153 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
11156 Register VReg1 =
MRI->createVirtualRegister(TRC);
11157 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRpci))
11161 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPr))
11167 BuildMI(DispatchBB, dl,
TII->get(ARM::tBcc))
11172 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
11173 BuildMI(DispContBB, dl,
TII->get(ARM::tLSLri), NewVReg2)
11179 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11180 BuildMI(DispContBB, dl,
TII->get(ARM::tLEApcrelJT), NewVReg3)
11184 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11185 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg4)
11195 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11196 BuildMI(DispContBB, dl,
TII->get(ARM::tLDRi), NewVReg5)
11202 unsigned NewVReg6 = NewVReg5;
11203 if (IsPositionIndependent) {
11204 NewVReg6 =
MRI->createVirtualRegister(TRC);
11205 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg6)
11212 BuildMI(DispContBB, dl,
TII->get(ARM::tBR_JTr))
11216 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11217 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRi12), NewVReg1)
11223 if (NumLPads < 256) {
11224 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPri))
11228 }
else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) {
11229 Register VReg1 =
MRI->createVirtualRegister(TRC);
11230 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVi16), VReg1)
11231 .
addImm(NumLPads & 0xFFFF)
11234 unsigned VReg2 = VReg1;
11235 if ((NumLPads & 0xFFFF0000) != 0) {
11236 VReg2 =
MRI->createVirtualRegister(TRC);
11237 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVTi16), VReg2)
11243 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11250 const Constant *
C = ConstantInt::get(Int32Ty, NumLPads);
11253 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
11256 Register VReg1 =
MRI->createVirtualRegister(TRC);
11257 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRcp))
11262 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11273 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11274 BuildMI(DispContBB, dl,
TII->get(ARM::MOVsi), NewVReg3)
11279 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11280 BuildMI(DispContBB, dl,
TII->get(ARM::LEApcrelJT), NewVReg4)
11287 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11288 BuildMI(DispContBB, dl,
TII->get(ARM::LDRrs), NewVReg5)
11295 if (IsPositionIndependent) {
11296 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTadd))
11301 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTr))
11310 if (SeenMBBs.
insert(CurMBB).second)
11322 while (!Successors.empty()) {
11331 BB->normalizeSuccProbs();
11338 II = BB->rbegin(), IE = BB->rend();
II != IE; ++
II) {
11339 if (!
II->isCall())
continue;
11343 OI =
II->operands_begin(), OE =
II->operands_end();
11345 if (!OI->isReg())
continue;
11346 DefRegs[OI->getReg()] =
true;
11351 for (
unsigned i = 0; SavedRegs[i] != 0; ++i) {
11352 unsigned Reg = SavedRegs[i];
11354 !ARM::tGPRRegClass.contains(Reg) &&
11355 !ARM::hGPRRegClass.contains(Reg))
11357 if (Subtarget->
isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11359 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11372 MBBLPad->setIsEHPad(
false);
11375 MI.eraseFromParent();
11388static unsigned getLdOpcode(
unsigned LdSize,
bool IsThumb1,
bool IsThumb2) {
11390 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11391 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11393 return LdSize == 4 ? ARM::tLDRi
11394 : LdSize == 2 ? ARM::tLDRHi
11395 : LdSize == 1 ? ARM::tLDRBi : 0;
11397 return LdSize == 4 ? ARM::t2LDR_POST
11398 : LdSize == 2 ? ARM::t2LDRH_POST
11399 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11400 return LdSize == 4 ? ARM::LDR_POST_IMM
11401 : LdSize == 2 ? ARM::LDRH_POST
11402 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11407static unsigned getStOpcode(
unsigned StSize,
bool IsThumb1,
bool IsThumb2) {
11409 return StSize == 16 ? ARM::VST1q32wb_fixed
11410 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11412 return StSize == 4 ? ARM::tSTRi
11413 : StSize == 2 ? ARM::tSTRHi
11414 : StSize == 1 ? ARM::tSTRBi : 0;
11416 return StSize == 4 ? ARM::t2STR_POST
11417 : StSize == 2 ? ARM::t2STRH_POST
11418 : StSize == 1 ? ARM::t2STRB_POST : 0;
11419 return StSize == 4 ? ARM::STR_POST_IMM
11420 : StSize == 2 ? ARM::STRH_POST
11421 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11428 unsigned LdSize,
unsigned Data,
unsigned AddrIn,
11429 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11430 unsigned LdOpc =
getLdOpcode(LdSize, IsThumb1, IsThumb2);
11431 assert(LdOpc != 0 &&
"Should have a load opcode");
11438 }
else if (IsThumb1) {
11444 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11449 }
else if (IsThumb2) {
11469 unsigned StSize,
unsigned Data,
unsigned AddrIn,
11470 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11471 unsigned StOpc =
getStOpcode(StSize, IsThumb1, IsThumb2);
11472 assert(StOpc != 0 &&
"Should have a store opcode");
11474 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11479 }
else if (IsThumb1) {
11486 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11491 }
else if (IsThumb2) {
11492 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11498 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11519 unsigned SizeVal =
MI.getOperand(2).getImm();
11520 unsigned Alignment =
MI.getOperand(3).getImm();
11525 unsigned UnitSize = 0;
11530 bool IsThumb2 = Subtarget->
isThumb2();
11531 bool IsThumb = Subtarget->isThumb();
11533 if (Alignment & 1) {
11535 }
else if (Alignment & 2) {
11540 Subtarget->hasNEON()) {
11541 if ((Alignment % 16 == 0) && SizeVal >= 16)
11543 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11552 bool IsNeon = UnitSize >= 8;
11553 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11555 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11556 : UnitSize == 8 ? &ARM::DPRRegClass
11559 unsigned BytesLeft = SizeVal % UnitSize;
11560 unsigned LoopSize = SizeVal - BytesLeft;
11562 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11566 unsigned srcIn = src;
11567 unsigned destIn = dest;
11568 for (
unsigned i = 0; i < LoopSize; i+=UnitSize) {
11569 Register srcOut =
MRI.createVirtualRegister(TRC);
11570 Register destOut =
MRI.createVirtualRegister(TRC);
11571 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11573 IsThumb1, IsThumb2);
11575 IsThumb1, IsThumb2);
11583 for (
unsigned i = 0; i < BytesLeft; i++) {
11584 Register srcOut =
MRI.createVirtualRegister(TRC);
11585 Register destOut =
MRI.createVirtualRegister(TRC);
11586 Register scratch =
MRI.createVirtualRegister(TRC);
11588 IsThumb1, IsThumb2);
11590 IsThumb1, IsThumb2);
11594 MI.eraseFromParent();
11620 MF->
insert(It, loopMBB);
11621 MF->
insert(It, exitMBB);
11624 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
11634 Register varEnd =
MRI.createVirtualRegister(TRC);
11636 BuildMI(BB, dl,
TII->get(IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11639 }
else if (Subtarget->genExecuteOnly()) {
11640 assert(IsThumb &&
"Non-thumb expected to have used movt");
11645 const Constant *
C = ConstantInt::get(Int32Ty, LoopSize);
11676 Register varLoop =
MRI.createVirtualRegister(TRC);
11677 Register varPhi =
MRI.createVirtualRegister(TRC);
11678 Register srcLoop =
MRI.createVirtualRegister(TRC);
11679 Register srcPhi =
MRI.createVirtualRegister(TRC);
11680 Register destLoop =
MRI.createVirtualRegister(TRC);
11681 Register destPhi =
MRI.createVirtualRegister(TRC);
11689 BuildMI(BB, dl,
TII->get(ARM::PHI), destPhi)
11695 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11697 IsThumb1, IsThumb2);
11699 IsThumb1, IsThumb2);
11703 BuildMI(*BB, BB->
end(), dl,
TII->get(ARM::tSUBi8), varLoop)
11711 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
11720 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11729 auto StartOfExit = exitMBB->
begin();
11733 unsigned srcIn = srcLoop;
11734 unsigned destIn = destLoop;
11735 for (
unsigned i = 0; i < BytesLeft; i++) {
11736 Register srcOut =
MRI.createVirtualRegister(TRC);
11737 Register destOut =
MRI.createVirtualRegister(TRC);
11738 Register scratch =
MRI.createVirtualRegister(TRC);
11739 emitPostLd(BB, StartOfExit,
TII, dl, 1, scratch, srcIn, srcOut,
11740 IsThumb1, IsThumb2);
11741 emitPostSt(BB, StartOfExit,
TII, dl, 1, scratch, destIn, destOut,
11742 IsThumb1, IsThumb2);
11747 MI.eraseFromParent();
11759 "__chkstk is only supported on Windows");
11760 assert(Subtarget->
isThumb2() &&
"Windows on ARM requires Thumb-2 mode");
11780 switch (
TM.getCodeModel()) {
11822 MI.eraseFromParent();
11846 .
addReg(
MI.getOperand(0).getReg())
11854 MI.eraseFromParent();
11878 if (miI == BB->
end()) {
11880 if (Succ->isLiveIn(ARM::CPSR))
11886 SelectItr->addRegisterKilled(ARM::CPSR,
TRI);
11898 Register AddDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11899 BuildMI(TpEntry, Dl,
TII->get(ARM::t2ADDri), AddDestReg)
11905 Register LsrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11906 BuildMI(TpEntry, Dl,
TII->get(ARM::t2LSRri), LsrDestReg)
11912 Register TotalIterationsReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11913 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg)
11916 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopStart))
11917 .
addUse(TotalIterationsReg)
11924 return TotalIterationsReg;
11935 Register TotalIterationsReg,
bool IsMemcpy) {
11942 SrcPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11943 CurrSrcReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11944 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), SrcPhiReg)
11952 Register DestPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11953 Register CurrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11954 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), DestPhiReg)
11961 Register LoopCounterPhiReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11962 Register RemainingLoopIterationsReg =
11963 MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11964 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), LoopCounterPhiReg)
11965 .
addUse(TotalIterationsReg)
11967 .
addUse(RemainingLoopIterationsReg)
11971 Register PredCounterPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11972 Register RemainingElementsReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11973 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), PredCounterPhiReg)
11974 .
addUse(ElementCountReg)
11976 .
addUse(RemainingElementsReg)
11980 Register VccrReg =
MRI.createVirtualRegister(&ARM::VCCRRegClass);
11981 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VCTP8), VccrReg)
11982 .
addUse(PredCounterPhiReg)
11987 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2SUBri), RemainingElementsReg)
11988 .
addUse(PredCounterPhiReg)
11996 SrcValueReg =
MRI.createVirtualRegister(&ARM::MQPRRegClass);
11997 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VLDRBU8_post))
12006 SrcValueReg = OpSrcReg;
12008 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VSTRBU8_post))
12019 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg)
12020 .
addUse(LoopCounterPhiReg)
12023 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopEnd))
12024 .
addUse(RemainingLoopIterationsReg)
12037 bool isThumb2 = Subtarget->
isThumb2();
12038 switch (
MI.getOpcode()) {
12045 case ARM::tLDR_postidx: {
12049 .
add(
MI.getOperand(2))
12050 .
add(
MI.getOperand(3))
12051 .
add(
MI.getOperand(4))
12052 .
add(
MI.getOperand(0))
12054 MI.eraseFromParent();
12058 case ARM::MVE_MEMCPYLOOPINST:
12059 case ARM::MVE_MEMSETLOOPINST: {
12089 Register OpDestReg =
MI.getOperand(0).getReg();
12090 Register OpSrcReg =
MI.getOperand(1).getReg();
12091 Register OpSizeReg =
MI.getOperand(2).getReg();
12111 if (TpExit == BB) {
12113 "block containing memcpy/memset Pseudo");
12126 bool IsMemcpy =
MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
12128 OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy);
12143 MI.eraseFromParent();
12153 case ARM::t2STR_preidx:
12154 MI.setDesc(
TII->get(ARM::t2STR_PRE));
12156 case ARM::t2STRB_preidx:
12157 MI.setDesc(
TII->get(ARM::t2STRB_PRE));
12159 case ARM::t2STRH_preidx:
12160 MI.setDesc(
TII->get(ARM::t2STRH_PRE));
12163 case ARM::STRi_preidx:
12164 case ARM::STRBi_preidx: {
12165 unsigned NewOpc =
MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
12166 : ARM::STRB_PRE_IMM;
12168 unsigned Offset =
MI.getOperand(4).getImm();
12176 .
add(
MI.getOperand(0))
12177 .
add(
MI.getOperand(1))
12178 .
add(
MI.getOperand(2))
12180 .
add(
MI.getOperand(5))
12181 .
add(
MI.getOperand(6))
12183 MI.eraseFromParent();
12186 case ARM::STRr_preidx:
12187 case ARM::STRBr_preidx:
12188 case ARM::STRH_preidx: {
12190 switch (
MI.getOpcode()) {
12192 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG;
break;
12193 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG;
break;
12194 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE;
break;
12199 MI.eraseFromParent();
12203 case ARM::tMOVCCr_pseudo: {
12221 F->insert(It, copy0MBB);
12222 F->insert(It, sinkMBB);
12225 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
12231 if (!
MI.killsRegister(ARM::CPSR,
nullptr) &&
12247 .
addImm(
MI.getOperand(3).getImm())
12248 .
addReg(
MI.getOperand(4).getReg());
12263 .
addReg(
MI.getOperand(1).getReg())
12265 .
addReg(
MI.getOperand(2).getReg())
12268 MI.eraseFromParent();
12273 case ARM::BCCZi64: {
12279 bool RHSisZero =
MI.getOpcode() == ARM::BCCZi64;
12284 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12288 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12294 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12298 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12308 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12317 MI.eraseFromParent();
12321 case ARM::Int_eh_sjlj_setjmp:
12322 case ARM::Int_eh_sjlj_setjmp_nofp:
12323 case ARM::tInt_eh_sjlj_setjmp:
12324 case ARM::t2Int_eh_sjlj_setjmp:
12325 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12328 case ARM::Int_eh_sjlj_setup_dispatch:
12329 EmitSjLjDispatchBlock(
MI, BB);
12352 Fn->
insert(BBI, SinkBB);
12354 Register ABSSrcReg =
MI.getOperand(1).getReg();
12355 Register ABSDstReg =
MI.getOperand(0).getReg();
12356 bool ABSSrcKIll =
MI.getOperand(1).isKill();
12357 bool isThumb2 = Subtarget->
isThumb2();
12361 Register NewRsbDstReg =
MRI.createVirtualRegister(
12362 isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass);
12376 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12383 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).
addMBB(SinkBB)
12390 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
12399 TII->get(ARM::PHI), ABSDstReg)
12404 MI.eraseFromParent();
12409 case ARM::COPY_STRUCT_BYVAL_I32:
12411 return EmitStructByval(
MI, BB);
12412 case ARM::WIN__CHKSTK:
12413 return EmitLowered__chkstk(
MI, BB);
12414 case ARM::WIN__DBZCHK:
12415 return EmitLowered__dbzchk(
MI, BB);
12432 if (!
Node->hasAnyUseOfValue(0)) {
12433 MI.getOperand(0).setIsDead(
true);
12435 if (!
Node->hasAnyUseOfValue(1)) {
12436 MI.getOperand(1).setIsDead(
true);
12440 for (
unsigned I = 0;
I !=
MI.getOperand(4).getImm(); ++
I) {
12441 Register TmpReg =
MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
12442 : &ARM::GPRRegClass);
12449 if (
MI.getOpcode() == ARM::MEMCPY) {
12467 MCID = &
TII->get(NewOpc);
12470 MI.getDesc().getNumOperands() + 5 -
MI.getDesc().getSize()
12471 &&
"converted opcode should be the same except for cc_out"
12472 " (and, on Thumb1, pred)");
12482 MI.addOperand(
MI.getOperand(1));
12483 MI.removeOperand(1);
12487 for (
unsigned i =
MI.getNumOperands(); i--;) {
12489 if (
op.isReg() &&
op.isUse()) {
12492 MI.tieOperands(DefIdx, i);
12506 if (!
MI.hasOptionalDef() || !MCID->
operands()[ccOutIdx].isOptionalDef()) {
12507 assert(!NewOpc &&
"Optional cc_out operand required");
12512 bool definesCPSR =
false;
12513 bool deadCPSR =
false;
12514 for (
unsigned i = MCID->
getNumOperands(), e =
MI.getNumOperands(); i != e;
12518 definesCPSR =
true;
12521 MI.removeOperand(i);
12525 if (!definesCPSR) {
12526 assert(!NewOpc &&
"Optional cc_out operand required");
12529 assert(deadCPSR == !Node->hasAnyUseOfValue(1) &&
"inconsistent dead flag");
12531 assert(!
MI.getOperand(ccOutIdx).getReg() &&
12532 "expect uninitialized optional cc_out operand");
12570 switch (
N->getOpcode()) {
12571 default:
return false;
12573 CC =
N->getOperand(0);
12595 EVT VT =
N->getValueType(0);
12596 CC =
N->getOperand(0);
12597 if (
CC.getValueType() != MVT::i1 ||
CC.getOpcode() !=
ISD::SETCC)
12643 EVT VT =
N->getValueType(0);
12646 bool SwapSelectOps;
12648 NonConstantVal, DAG))
12654 OtherOp, NonConstantVal);
12660 CCOp, TrueVal, FalseVal);
12684 if (
N->getOpcode() ==
ARMISD::VTRN &&
N->getValueType(0) == MVT::v2i32)
12699 if (!
N->getValueType(0).is64BitVector())
12707 EVT VT =
N->getValueType(0);
12746 EVT VT =
N->getValueType(0);
12752 Opcode = Intrinsic::arm_neon_vpaddls;
12754 Opcode = Intrinsic::arm_neon_vpaddlu;
12782 EVT VT =
N->getValueType(0);
12797 unsigned nextIndex = 0;
12865 return DAG.
getNode(ExtOp, dl, VT, tmp);
12896 if (SRA.getOpcode() !=
ISD::SRA) {
12902 if (
auto Const = dyn_cast<ConstantSDNode>(SRA.getOperand(1))) {
12903 if (Const->getZExtValue() != 31)
12908 if (SRA.getOperand(0) !=
Mul)
12912 SDLoc dl(AddcNode);
12913 unsigned Opcode = 0;
12919 Op0 =
Mul.getOperand(0);
12920 Op1 =
Mul.getOperand(1);
12923 Op0 =
Mul.getOperand(0);
12924 Op1 =
Mul.getOperand(1).getOperand(0);
12927 Op0 =
Mul.getOperand(0).getOperand(0);
12928 Op1 =
Mul.getOperand(1);
12931 Op0 =
Mul->getOperand(0).getOperand(0);
12932 Op1 =
Mul->getOperand(1).getOperand(0);
12941 SDValue HiMLALResult(SMLAL.getNode(), 1);
12942 SDValue LoMLALResult(SMLAL.getNode(), 0);
12948 SDValue resNode(AddcNode, 0);
12977 "Expect an ADDE or SUBE");
12981 "ADDE node has the wrong inputs");
13000 "Expect ADDC with two result values. First: i32");
13020 bool IsLeftOperandMUL =
false;
13025 IsLeftOperandMUL =
true;
13036 SDValue *LowAddSub =
nullptr;
13039 if ((AddeSubeOp0 != MULOp.
getValue(1)) && (AddeSubeOp1 != MULOp.
getValue(1)))
13042 if (IsLeftOperandMUL)
13043 HiAddSub = &AddeSubeOp1;
13045 HiAddSub = &AddeSubeOp0;
13050 if (AddcSubcOp0 == MULOp.
getValue(0)) {
13051 LoMul = &AddcSubcOp0;
13052 LowAddSub = &AddcSubcOp1;
13054 if (AddcSubcOp1 == MULOp.
getValue(0)) {
13055 LoMul = &AddcSubcOp1;
13056 LowAddSub = &AddcSubcOp0;
13064 if (AddcSubcNode == HiAddSub->getNode() ||
13080 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->
useMulOps() &&
13094 return SDValue(AddeSubeNode, 0);
13105 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13115 return SDValue(AddeSubeNode, 0);
13127 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13136 SDNode *UmlalNode =
nullptr;
13161 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13175 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13180 SDNode* AddcNode =
N->getOperand(2).getNode();
13181 SDNode* AddeNode =
N->getOperand(3).getNode();
13189 {N->getOperand(0), N->getOperand(1),
13190 AddcNode->getOperand(0), AddcNode->getOperand(1)});
13214 int32_t imm =
C->getSExtValue();
13215 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13220 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
N->getOperand(0),
RHS);
13235 int64_t imm =
C->getSExtValue();
13246 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
13247 N->getOperand(0),
RHS,
N->getOperand(2));
13259 if (!Subtarget->hasMVEIntegerOps())
13272 SetCC =
N->getOperand(0);
13275 CC = cast<CondCodeSDNode>(SetCC->
getOperand(2))->get();
13276 TrueVal =
N->getOperand(1);
13277 FalseVal =
N->getOperand(2);
13279 LHS =
N->getOperand(0);
13280 RHS =
N->getOperand(1);
13281 CC = cast<CondCodeSDNode>(
N->getOperand(4))->get();
13282 TrueVal =
N->getOperand(2);
13283 FalseVal =
N->getOperand(3);
13288 unsigned int Opcode = 0;
13317 switch (TrueVal->getOpcode()) {
13336 if (TrueVal !=
LHS || FalseVal !=
RHS)
13339 EVT LeftType =
LHS->getValueType(0);
13340 EVT RightType =
RHS->getValueType(0);
13343 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13347 if (VectorScalarType != MVT::i32)
13355 if (VectorScalarType != MVT::i32)
13368 EVT VT =
N->getValueType(0);
13376 Shft =
N->getOperand(0);
13382 cast<CondCodeSDNode>(Cmp.getOperand(2))->get() !=
ISD::SETLT ||
13383 Cmp.getOperand(0) !=
N->getOperand(1) ||
13384 Cmp.getOperand(1) !=
N->getOperand(2))
13386 Shft =
N->getOperand(1);
13398 ScalarType = MVT::i8;
13401 case (1 << 15) - 1:
13402 ScalarType = MVT::i16;
13405 case (1ULL << 31) - 1:
13406 ScalarType = MVT::i32;
13437 unsigned LegalLanes = 128 / (ShftAmt + 1);
13461 for (
unsigned I = 0;
I < NumParts; ++
I) {
13478 if (!Subtarget->hasMVEIntegerOps())
13493 if (
N->getOperand(0).getOpcode() !=
ISD::XOR)
13503 if (!Const || !Const->isOne())
13521 EVT VT =
N->getValueType(0);
13523 if (!Subtarget->hasMVEIntegerOps() ||
13552 Opc = Intrinsic::arm_mve_vctp64;
13555 Opc = Intrinsic::arm_mve_vctp32;
13558 Opc = Intrinsic::arm_mve_vctp16;
13561 Opc = Intrinsic::arm_mve_vctp8;
13615 EVT VT =
N->getValueType(0);
13621 switch (
Op.getOpcode()) {
13638 !isa<ConstantSDNode>(N0) && N1->
hasOneUse()) {
13646 unsigned N0RedOp = 0;
13653 unsigned N1RedOp = 0;
13667 if (
SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13669 if (
SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13676 auto DistrubuteVecReduceLoad = [&](
SDValue N0,
SDValue N1,
bool IsForward) {
13690 LoadSDNode *Load0 = dyn_cast<LoadSDNode>(N0);
13691 LoadSDNode *Load1 = dyn_cast<LoadSDNode>(N1);
13700 if (!BaseLocDecomp0.getBase() ||
13701 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13702 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13704 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13706 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13716 if (IsBefore < 0) {
13719 }
else if (IsBefore > 0) {
13732 }
else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13742 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13752 if (
SDValue R = DistrubuteVecReduceLoad(N0, N1,
true))
13754 if (
SDValue R = DistrubuteVecReduceLoad(N1, N0,
false))
13761 if (!Subtarget->hasMVEIntegerOps())
13767 EVT VT =
N->getValueType(0);
13772 if (VT != MVT::i64)
13783 auto MakeVecReduce = [&](
unsigned Opcode,
unsigned OpcodeA,
SDValue NA,
13801 std::tie(Ops[0], Ops[1]) = DAG.
SplitScalar(NA, dl, MVT::i32, MVT::i32);
13803 unsigned S = VecRed->
getOpcode() == OpcodeA ? 2 : 0;
13852 "Expected shift op");
13854 SDValue ShiftLHS =
N->getOperand(0);
13878 if (
auto *Const = dyn_cast<ConstantSDNode>(N1->
getOperand(1))) {
13879 if (Const->getAPIntValue().ult(256))
13882 Const->getAPIntValue().sgt(-256))
13898 (
N->getOperand(0).getOpcode() ==
ISD::SHL ||
13899 N->getOperand(0).getOpcode() ==
ISD::SRL) &&
13900 "Expected XOR(SHIFT) pattern");
13903 auto *XorC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13904 auto *ShiftC = dyn_cast<ConstantSDNode>(
N->getOperand(0).getOperand(1));
13905 if (XorC && ShiftC) {
13906 unsigned MaskIdx, MaskLen;
13907 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13908 unsigned ShiftAmt = ShiftC->getZExtValue();
13909 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
13910 if (
N->getOperand(0).getOpcode() ==
ISD::SHL)
13911 return MaskIdx == ShiftAmt && MaskLen == (
BitWidth - ShiftAmt);
13912 return MaskIdx == 0 && MaskLen == (
BitWidth - ShiftAmt);
13922 N->getOperand(0).getOpcode() ==
ISD::SRL) ||
13924 N->getOperand(0).getOpcode() ==
ISD::SHL)) &&
13925 "Expected shift-shift mask");
13938 return Subtarget->hasMVEIntegerOps() &&
isTypeLegal(VT);
13942 if (!Subtarget->hasNEON()) {
13961 return Subtarget->hasFP64();
13964 return Subtarget->hasMVEFloatOps();
13993 if (ST->isThumb() && ST->isThumb1Only())
13997 for (
auto *U :
N->users()) {
13998 switch(U->getOpcode()) {
14011 if (isa<ConstantSDNode>(U->getOperand(0)) ||
14012 isa<ConstantSDNode>(U->getOperand(1)))
14016 if (U->getOperand(0).getOpcode() ==
ISD::SHL ||
14017 U->getOperand(1).getOpcode() ==
ISD::SHL)
14027 if (
N->getOperand(0).getOpcode() !=
ISD::SHL)
14032 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
14033 auto *C2 = dyn_cast<ConstantSDNode>(SHL.getOperand(1));
14034 if (!C1ShlC2 || !C2)
14037 APInt C2Int = C2->getAPIntValue();
14038 APInt C1Int = C1ShlC2->getAPIntValue();
14040 if (C2Int.
uge(C2Width))
14046 if ((C1Int & Mask) != C1Int)
14053 auto LargeImm = [](
const APInt &Imm) {
14054 unsigned Zeros = Imm.countl_zero() + Imm.countr_zero();
14055 return Imm.getBitWidth() - Zeros > 8;
14058 if (LargeImm(C1Int) || LargeImm(C2Int))
14070 SHL.dump();
N->dump());
14105 if (CSINC.getOpcode() !=
ARMISD::CSINC || !CSINC.hasOneUse())
14114 CSINC.getOperand(0)),
14115 CSINC.getOperand(1), CSINC.getOperand(2),
14116 CSINC.getOperand(3));
14135 if (!Subtarget->hasMVEIntegerOps() || !
N->getValueType(0).isVector())
14177 if (!Subtarget->hasVMLxForwarding())
14196 EVT VT =
N->getValueType(0);
14207 EVT VT =
N->getValueType(0);
14208 if (VT != MVT::v2i64)
14217 EVT VT = cast<VTSDNode>(
Op->getOperand(1))->getVT();
14219 return Op->getOperand(0);
14233 And =
And->getOperand(0);
14238 Mask = Mask->getOperand(0);
14241 Mask.getValueType() != MVT::v4i32)
14247 return And->getOperand(0);
14252 if (
SDValue Op0 = IsSignExt(N0)) {
14253 if (
SDValue Op1 = IsSignExt(N1)) {
14259 if (
SDValue Op0 = IsZeroExt(N0)) {
14260 if (
SDValue Op1 = IsZeroExt(N1)) {
14275 EVT VT =
N->getValueType(0);
14276 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14287 if (VT != MVT::i32)
14294 int64_t MulAmt =
C->getSExtValue();
14295 unsigned ShiftAmt = llvm::countr_zero<uint64_t>(MulAmt);
14297 ShiftAmt = ShiftAmt & (32 - 1);
14302 MulAmt >>= ShiftAmt;
14305 if (llvm::has_single_bit<uint32_t>(MulAmt - 1)) {
14313 }
else if (llvm::has_single_bit<uint32_t>(MulAmt + 1)) {
14325 if (llvm::has_single_bit<uint32_t>(MulAmtAbs + 1)) {
14333 }
else if (llvm::has_single_bit<uint32_t>(MulAmtAbs - 1)) {
14363 if (
N->getValueType(0) != MVT::i32)
14372 if (C1 == 255 || C1 == 65535)
14375 SDNode *N0 =
N->getOperand(0).getNode();
14389 if (!C2 || C2 >= 32)
14433 if (Trailing == C2 && C2 + C3 < 32) {
14446 if (Leading == C2 && C2 + C3 < 32) {
14474 EVT VT =
N->getValueType(0);
14478 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14481 APInt SplatBits, SplatUndef;
14482 unsigned SplatBitSize;
14484 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14485 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14486 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14487 SplatBitSize == 64) {
14521 if (!Subtarget->hasV6Ops() ||
14522 (Subtarget->isThumb() &&
14523 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14526 SDValue SRL = OR->getOperand(0);
14527 SDValue SHL = OR->getOperand(1);
14530 SRL = OR->getOperand(1);
14531 SHL = OR->getOperand(0);
14538 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) ||
14542 SDNode *SMULLOHI = SRL.getOperand(0).getNode();
14543 if (SRL.getOperand(0) !=
SDValue(SMULLOHI, 0) ||
14544 SHL.getOperand(0) !=
SDValue(SMULLOHI, 1))
14563 unsigned Opcode = 0;
14564 if (
isS16(OpS16, DAG))
14582 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6T2Ops())
14585 EVT VT =
N->getValueType(0);
14600 if (VT != MVT::i32)
14613 if (Mask == 0xffff)
14620 if ((Val & ~Mask) != Val)
14645 (Mask == ~Mask2)) {
14648 if (Subtarget->hasDSP() &&
14649 (Mask == 0xffff || Mask == 0xffff0000))
14662 (~Mask == Mask2)) {
14665 if (Subtarget->hasDSP() &&
14666 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14739 EVT VT =
N->getValueType(0);
14744 auto IsFreelyInvertable = [&](
SDValue V) {
14751 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14767 EVT VT =
N->getValueType(0);
14773 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14774 VT == MVT::v8i1 || VT == MVT::v16i1))
14777 APInt SplatBits, SplatUndef;
14778 unsigned SplatBitSize;
14780 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14781 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14782 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14783 SplatBitSize == 64) {
14819 unsigned SplatBitSize;
14822 APInt SplatBits0, SplatBits1;
14826 if (BVN0 && BVN0->
isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
14827 HasAnyUndefs) && !HasAnyUndefs) {
14828 if (BVN1 && BVN1->
isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
14829 HasAnyUndefs) && !HasAnyUndefs) {
14834 SplatBits0 == ~SplatBits1) {
14864 EVT VT =
N->getValueType(0);
14879 if (Subtarget->hasMVEIntegerOps()) {
14910 ToMask = ~N->getConstantOperandAPInt(2);
14916 isa<ConstantSDNode>(
From->getOperand(1))) {
14917 APInt Shift =
From->getConstantOperandAPInt(1);
14930 unsigned LastActiveBitInA =
A.countr_zero();
14931 unsigned FirstActiveBitInB =
B.getBitWidth() -
B.countl_zero() - 1;
14932 return LastActiveBitInA - 1 == FirstActiveBitInB;
14937 APInt ToMask, FromMask;
14945 APInt NewToMask, NewFromMask;
14947 if (NewFrom !=
From)
14951 if ((NewToMask & ToMask).getBoolValue())
14976 unsigned InvMask =
N->getConstantOperandVal(2);
14978 unsigned Width = llvm::bit_width<unsigned>(~InvMask) - LSB;
14980 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14981 "undefined behavior");
14982 unsigned Mask = (1u << Width) - 1;
14984 if ((Mask & (~Mask2)) == 0)
14986 N->getOperand(0), N1.
getOperand(0),
N->getOperand(2));
14993 APInt ToMask1, FromMask1;
14996 APInt ToMask2, FromMask2;
15002 APInt NewFromMask = FromMask1 | FromMask2;
15003 APInt NewToMask = ToMask1 | ToMask2;
15005 EVT VT =
N->getValueType(0);
15008 if (NewFromMask[0] == 0)
15020 APInt ToMask1 = ~N->getConstantOperandAPInt(2);
15021 APInt ToMask2 = ~N0.getConstantOperandAPInt(2);
15023 if (!N0.
hasOneUse() || (ToMask1 & ToMask2) != 0 ||
15027 EVT VT =
N->getValueType(0);
15030 N->getOperand(1),
N->getOperand(2));
15044 SDValue CSInc = Cmp->getOperand(0);
15094 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ)
15095 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15098 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
15100 N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15113 SDValue InDouble =
N->getOperand(0);
15122 !cast<LoadSDNode>(InNode)->isVolatile()) {
15128 SDValue BasePtr = LD->getBasePtr();
15130 DAG.
getLoad(MVT::i32,
DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
15131 LD->getAlign(), LD->getMemOperand()->getFlags());
15137 LD->getPointerInfo().getWithOffset(4),
15139 LD->getMemOperand()->getFlags());
15151 isa<ConstantSDNode>(InDouble.
getOperand(1))) {
15172 if (!Subtarget->
isLittle() && BVSwap)
15182 if (isa<ConstantSDNode>(BV.
getOperand(2))) {
15190 if (!Subtarget->
isLittle() && BVSwap)
15235 if (Copy.getValueType() == MVT::f32 &&
15237 bool HasGlue = Copy->getNumOperands() == 3;
15238 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1),
15239 HasGlue ? Copy->getOperand(2) :
SDValue()};
15240 EVT OutTys[] = {
N->getValueType(0), MVT::Other, MVT::Glue};
15258 if (
LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Op0)) {
15259 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15260 LN0->getMemoryVT() == MVT::i16) {
15263 LN0->getBasePtr(), LN0->getMemOperand());
15281 EVT VT =
N->getValueType(0);
15315 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
15316 for (
unsigned i = 0; i < NumElts; ++i) {
15317 SDNode *Elt =
N->getOperand(i).getNode();
15334 if (
N->getNumOperands() == 2)
15340 EVT VT =
N->getValueType(0);
15346 for (
unsigned i = 0; i < NumElts; ++i) {
15372 EVT VT =
N->getValueType(0);
15380 assert(EltVT == MVT::f32 &&
"Unexpected type!");
15385 Use->getValueType(0).isFloatingPoint())
15393 unsigned NumOfBitCastedElts = 0;
15395 unsigned NumOfRelevantElts = NumElts;
15396 for (
unsigned Idx = 0;
Idx < NumElts; ++
Idx) {
15401 ++NumOfBitCastedElts;
15402 }
else if (Elt.
isUndef() || isa<ConstantSDNode>(Elt))
15405 --NumOfRelevantElts;
15409 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15427 for (
unsigned Idx = 0 ;
Idx < NumElts; ++
Idx) {
15432 V->getOperand(0).getValueType() == MVT::i32)
15434 V = V.getOperand(0);
15451 EVT VT =
N->getValueType(0);
15458 if (
Op->getOperand(0).getValueType() == VT)
15459 return Op->getOperand(0);
15474 if (
Op.getValueType() == MVT::i32) {
15485 EVT VT =
N->getValueType(0);
15490 if (ST->isLittle())
15494 if (
Op.getValueType() == VT)
15503 if (
Op->getOperand(0).getValueType() == VT)
15504 return Op->getOperand(0);
15513 if (!Subtarget->hasMVEIntegerOps())
15516 EVT VT =
N->getValueType(0);
15547 EVT VT =
N->getValueType(0);
15548 SDNode *Elt =
N->getOperand(1).getNode();
15563 Vec, V,
N->getOperand(2));
15573 EVT VT =
N->getValueType(0);
15582 Ext.getOperand(0).getValueType() == MVT::f32)
15583 Ext = Ext.getOperand(0);
15585 !isa<ConstantSDNode>(Ext.getOperand(1)) ||
15586 Ext.getConstantOperandVal(1) % 2 != 0)
15588 if (Ext->hasOneUse() && (Ext->user_begin()->getOpcode() ==
ISD::SINT_TO_FP ||
15592 SDValue Op0 = Ext.getOperand(0);
15595 unsigned Lane = Ext.getConstantOperandVal(1);
15601 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15602 isa<ConstantSDNode>(V->getOperand(1)) &&
15603 V->getConstantOperandVal(1) == Lane + 1 &&
15604 V->getOperand(0).getResNo() == ResNo;
15606 if (OtherIt == Op0->
users().end())
15611 SDValue OtherExt(*OtherIt, 0);
15624 DCI.
DAG.
getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32));
15636 EVT VT =
N->getValueType(0);
15642 if (VT == MVT::f16 &&
X.getValueType() == MVT::i32)
15644 if (VT == MVT::i32 &&
X.getValueType() == MVT::f16)
15646 if (VT == MVT::f32 &&
X.getValueType() == MVT::i32)
15649 while (
X.getValueType() != VT &&
X->getOpcode() ==
ISD::BITCAST)
15650 X =
X->getOperand(0);
15651 if (
X.getValueType() == VT)
15657 isa<ConstantSDNode>(
N->getOperand(1)) &&
15659 return Op0.
getOperand(
N->getConstantOperandVal(1));
15664 isa<ConstantSDNode>(
N->getOperand(1)) &&
15669 unsigned Offset =
N->getConstantOperandVal(1);
15681 unsigned Idx =
N->getConstantOperandVal(1);
15695 EVT VT =
N->getValueType(0);
15699 cast<VTSDNode>(
N->getOperand(1))->getVT() ==
15700 Op.getOperand(0).getValueType().getScalarType())
15710 SDValue SubVec =
N->getOperand(1);
15711 uint64_t IdxVal =
N->getConstantOperandVal(2);
15722 if (IdxVal == 0 && Vec.
isUndef())
15728 (IdxVal != 0 && IdxVal != NumSubElts))
15801 EVT VT =
N->getValueType(0);
15812 unsigned HalfElts = NumElts/2;
15814 for (
unsigned n = 0; n < NumElts; ++n) {
15817 if (MaskElt < (
int)HalfElts)
15819 else if (MaskElt >= (
int)NumElts && MaskElt < (
int)(NumElts + HalfElts))
15820 NewElt = HalfElts + MaskElt - NumElts;
15847 bool SimpleConstIncOnly,
15855 bool isLoadOp =
true;
15856 bool isLaneOp =
false;
15859 bool hasAlignment =
true;
15860 unsigned NewOpc = 0;
15861 unsigned NumVecs = 0;
15862 if (
Target.isIntrinsic) {
15863 unsigned IntNo =
N->getConstantOperandVal(1);
15867 case Intrinsic::arm_neon_vld1:
15871 case Intrinsic::arm_neon_vld2:
15875 case Intrinsic::arm_neon_vld3:
15879 case Intrinsic::arm_neon_vld4:
15883 case Intrinsic::arm_neon_vld1x2:
15886 hasAlignment =
false;
15888 case Intrinsic::arm_neon_vld1x3:
15891 hasAlignment =
false;
15893 case Intrinsic::arm_neon_vld1x4:
15896 hasAlignment =
false;
15898 case Intrinsic::arm_neon_vld2dup:
15902 case Intrinsic::arm_neon_vld3dup:
15906 case Intrinsic::arm_neon_vld4dup:
15910 case Intrinsic::arm_neon_vld2lane:
15915 case Intrinsic::arm_neon_vld3lane:
15920 case Intrinsic::arm_neon_vld4lane:
15925 case Intrinsic::arm_neon_vst1:
15930 case Intrinsic::arm_neon_vst2:
15935 case Intrinsic::arm_neon_vst3:
15940 case Intrinsic::arm_neon_vst4:
15945 case Intrinsic::arm_neon_vst2lane:
15951 case Intrinsic::arm_neon_vst3lane:
15957 case Intrinsic::arm_neon_vst4lane:
15963 case Intrinsic::arm_neon_vst1x2:
15967 hasAlignment =
false;
15969 case Intrinsic::arm_neon_vst1x3:
15973 hasAlignment =
false;
15975 case Intrinsic::arm_neon_vst1x4:
15979 hasAlignment =
false;
15984 switch (
N->getOpcode()) {
16020 VecTy =
N->getValueType(0);
16021 }
else if (
Target.isIntrinsic) {
16022 VecTy =
N->getOperand(
Target.AddrOpIdx + 1).getValueType();
16025 "Node has to be a load, a store, or an intrinsic!");
16026 VecTy =
N->getOperand(1).getValueType();
16034 if (isLaneOp || isVLDDUPOp)
16037 if (NumBytes >= 3 * 16 &&
User.ConstInc != NumBytes) {
16043 if (SimpleConstIncOnly &&
User.ConstInc != NumBytes)
16049 EVT AlignedVecTy = VecTy;
16066 if (isa<LSBaseSDNode>(
N)) {
16069 assert(NumVecs == 1 &&
"Unexpected multi-element generic load/store.");
16070 assert(!isLaneOp &&
"Unexpected generic load/store lane.");
16081 Alignment =
Align(1);
16087 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16089 for (n = 0; n < NumResultVecs; ++n)
16090 Tys[n] = AlignedVecTy;
16091 Tys[n++] = MVT::i32;
16092 Tys[n] = MVT::Other;
16107 unsigned LastOperand =
16108 hasAlignment ?
N->getNumOperands() - 1 :
N->getNumOperands();
16109 for (
unsigned i =
Target.AddrOpIdx + 1; i < LastOperand; ++i)
16118 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::STORE) {
16129 for (
unsigned i = 0; i < NumResultVecs; ++i)
16134 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::LOAD) {
16135 SDValue &LdVal = NewResults[0];
16171 switch (
N->getOpcode()) {
16174 if (isa<ConstantSDNode>(
N->getOperand(1))) {
16175 *
Ptr =
N->getOperand(0);
16176 *CInc =
N->getOperand(1);
16182 if (isa<ConstantSDNode>(
N->getOperand(2))) {
16183 *
Ptr =
N->getOperand(1);
16184 *CInc =
N->getOperand(2);
16233 unsigned ConstInc =
16255 unsigned UserOffset =
16258 if (!UserOffset || UserOffset <=
Offset)
16261 unsigned NewConstInc = UserOffset -
Offset;
16271 unsigned NumValidUpd = BaseUpdates.
size();
16272 for (
unsigned I = 0;
I < NumValidUpd;) {
16276 std::swap(BaseUpdates[
I], BaseUpdates[NumValidUpd]);
16284 BaseUpdates.
resize(NumValidUpd);
16289 std::stable_sort(BaseUpdates.
begin(), BaseUpdates.
end(),
16291 return LHS.ConstInc < RHS.ConstInc;
16320 unsigned IntNo =
N->getConstantOperandVal(1);
16321 if (IntNo == Intrinsic::arm_mve_vst2q &&
N->getConstantOperandVal(5) != 1)
16323 if (IntNo == Intrinsic::arm_mve_vst4q &&
N->getConstantOperandVal(7) != 3)
16345 bool isLoadOp =
true;
16346 unsigned NewOpc = 0;
16347 unsigned NumVecs = 0;
16351 case Intrinsic::arm_mve_vld2q:
16355 case Intrinsic::arm_mve_vld4q:
16359 case Intrinsic::arm_mve_vst2q:
16364 case Intrinsic::arm_mve_vst4q:
16374 VecTy =
N->getValueType(0);
16376 VecTy =
N->getOperand(3).getValueType();
16390 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16392 for (n = 0; n < NumResultVecs; ++n)
16394 Tys[n++] = MVT::i32;
16395 Tys[n] = MVT::Other;
16404 for (
unsigned i = 3; i <
N->getNumOperands(); ++i)
16412 for (
unsigned i = 0; i < NumResultVecs; ++i)
16431 EVT VT =
N->getValueType(0);
16437 SDNode *VLD =
N->getOperand(0).getNode();
16440 unsigned NumVecs = 0;
16441 unsigned NewOpc = 0;
16443 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16446 }
else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16449 }
else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16461 if (
Use.getResNo() == NumVecs)
16465 VLDLaneNo !=
User->getConstantOperandVal(1))
16472 for (n = 0; n < NumVecs; ++n)
16474 Tys[n] = MVT::Other;
16484 unsigned ResNo =
Use.getResNo();
16486 if (ResNo == NumVecs)
16493 std::vector<SDValue> VLDDupResults;
16494 for (
unsigned n = 0; n < NumVecs; ++n)
16508 EVT VT =
N->getValueType(0);
16511 if (Subtarget->hasMVEIntegerOps()) {
16515 ExtractVT = MVT::i32;
16517 N->getOperand(0),
N->getOperand(1));
16529 Op =
Op.getOperand(0);
16534 unsigned EltSize =
Op.getScalarValueSizeInBits();
16536 unsigned Imm =
Op.getConstantOperandVal(0);
16552 if (Subtarget->hasMVEIntegerOps()) {
16555 if (
Op.getValueType() == MVT::f32)
16558 else if (
Op.getValueType() == MVT::f16)
16563 if (!Subtarget->hasNEON())
16570 if (LD &&
Op.hasOneUse() && LD->isUnindexed() &&
16571 LD->getMemoryVT() ==
N->getValueType(0).getVectorElementType()) {
16572 SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1),
16577 LD->getMemoryVT(), LD->getMemOperand());
16588 EVT VT =
N->getValueType(0);
16610 assert(StVT != VT &&
"Cannot truncate to the same type");
16620 if (0 != (NumElems * FromEltSz) % ToEltSz)
16623 unsigned SizeRatio = FromEltSz / ToEltSz;
16628 NumElems * SizeRatio);
16634 for (
unsigned i = 0; i < NumElems; ++i)
16648 MVT StoreType = MVT::i8;
16650 if (TLI.
isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16669 unsigned E = (ToEltSz * NumElems) / StoreType.
getSizeInBits();
16670 for (
unsigned I = 0;
I < E;
I++) {
16701 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16704 unsigned NumElements = 4;
16721 unsigned Off0 = Rev ? NumElts : 0;
16722 unsigned Off1 = Rev ? 0 : NumElts;
16724 for (
unsigned I = 0;
I < NumElts;
I += 2) {
16725 if (M[
I] >= 0 && M[
I] != (
int)(Off0 +
I / 2))
16727 if (M[
I + 1] >= 0 && M[
I + 1] != (
int)(Off1 +
I / 2))
16734 if (
auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Trunc.
getOperand(0)))
16735 if (isVMOVNShuffle(Shuffle,
false) || isVMOVNShuffle(Shuffle,
true))
16755 unsigned NewOffset = i * NumElements * ToEltVT.
getSizeInBits() / 8;
16770 NewToVT, Alignment, MMOFlags, AAInfo);
16803 unsigned NewOffset =
16811 NewToVT, Alignment, MMOFlags, AAInfo);
16833 {Extract.getOperand(0), Extract.getOperand(1)});
16864 if (Subtarget->hasNEON())
16868 if (Subtarget->hasMVEFloatOps())
16872 if (Subtarget->hasMVEIntegerOps()) {
16948 if (!Subtarget->hasNEON())
16952 if (!
Op.getValueType().isVector() || !
Op.getValueType().isSimple() ||
16957 if (!isa<BuildVectorSDNode>(ConstVec))
16960 MVT FloatTy =
Op.getSimpleValueType().getVectorElementType();
16962 MVT IntTy =
N->getSimpleValueType(0).getVectorElementType();
16964 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
16965 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
16976 if (
C == -1 ||
C == 0 ||
C > 32)
16981 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
16982 Intrinsic::arm_neon_vcvtfp2fxu;
16985 DAG.
getConstant(IntrinsicOpcode, dl, MVT::i32),
Op->getOperand(0),
16988 if (IntBits < FloatBits)
16996 if (!Subtarget->hasMVEFloatOps())
17004 EVT VT =
N->getValueType(0);
17009 auto isIdentitySplat = [&](
SDValue Op,
bool NSZ) {
17013 uint64_t ImmVal =
Op.getOperand(0).getConstantOperandVal(0);
17014 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
17016 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
17029 if (!isIdentitySplat(Op1.
getOperand(2), NSZ))
17040 EVT VT =
N->getValueType(0);
17043 if (!
N->getFlags().hasAllowReassociation())
17050 unsigned Opc =
A.getConstantOperandVal(0);
17051 if (Opc != Intrinsic::arm_mve_vcmlaq)
17056 A.getOperand(3),
A.getOperand(4));
17088 if (!Subtarget->hasNEON())
17092 unsigned OpOpcode =
Op.getNode()->getOpcode();
17093 if (!
N->getValueType(0).isVector() || !
N->getValueType(0).isSimple() ||
17097 SDValue ConstVec =
N->getOperand(1);
17098 if (!isa<BuildVectorSDNode>(ConstVec))
17101 MVT FloatTy =
N->getSimpleValueType(0).getVectorElementType();
17103 MVT IntTy =
Op.getOperand(0).getSimpleValueType().getVectorElementType();
17105 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
17106 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17126 int32_t
C = IntVal.exactLogBase2();
17127 if (
C == -1 ||
C == 0 ||
C > 32)
17133 if (IntBits < FloatBits)
17135 NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, ConvInput);
17137 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfxs2fp
17138 : Intrinsic::arm_neon_vcvtfxu2fp;
17146 if (!ST->hasMVEIntegerOps())
17150 EVT ResVT =
N->getValueType(0);
17178 EVT AVT =
A.getValueType();
17184 auto ExtendIfNeeded = [&](
SDValue A,
unsigned ExtendCode) {
17185 EVT AVT =
A.getValueType();
17197 if (ExtTypeMatches(
A, ExtTypes))
17198 return ExtendIfNeeded(
A, ExtendCode);
17201 auto IsPredVADDV = [&](
MVT RetTy,
unsigned ExtendCode,
17208 if (Ext->getOpcode() != ExtendCode)
17211 if (ExtTypeMatches(
A, ExtTypes))
17212 return ExtendIfNeeded(
A, ExtendCode);
17225 if (ResVT !=
RetTy)
17228 if (
Mul->getOpcode() == ExtendCode &&
17229 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17231 Mul =
Mul->getOperand(0);
17240 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17241 A = ExtendIfNeeded(
A, ExtendCode);
17242 B = ExtendIfNeeded(
B, ExtendCode);
17260 if (
Mul->getOpcode() == ExtendCode &&
17261 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17263 Mul =
Mul->getOperand(0);
17272 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17273 A = ExtendIfNeeded(
A, ExtendCode);
17274 B = ExtendIfNeeded(
B, ExtendCode);
17285 EVT VT = Ops[0].getValueType();
17286 if (VT == MVT::v16i8) {
17288 "Unexpected illegal long reduction opcode");
17293 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
17296 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
17317 if (IsVMLAV(MVT::i64,
ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17320 if (IsVMLAV(MVT::i64,
ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17384 Op =
Op->getOperand(1);
17386 Op->getOperand(0)->getOpcode() ==
ISD::MUL) {
17388 if (
Mul->getOperand(0) ==
Mul->getOperand(1) &&
17405 unsigned VecOp =
N->getOperand(0).getValueType().isVector() ? 0 : 2;
17406 auto *Shuf = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp));
17407 if (!Shuf || !Shuf->getOperand(1).isUndef())
17412 APInt SetElts(Mask.size(), 0);
17413 for (
int E : Mask) {
17414 if (E < 0 || E >= (
int)Mask.size())
17421 if (
N->getNumOperands() != VecOp + 1) {
17422 auto *Shuf2 = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp + 1));
17423 if (!Shuf2 || !Shuf2->getOperand(1).isUndef() || Shuf2->getMask() != Mask)
17429 if (
Op.getValueType().isVector())
17441 unsigned IsTop =
N->getConstantOperandVal(2);
17448 if (Op0->
isUndef() && !IsTop)
17462 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17464 APInt Op0DemandedElts =
17465 IsTop ? Op1DemandedElts
17480 unsigned IsTop =
N->getConstantOperandVal(2);
17482 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17483 APInt Op0DemandedElts =
17495 EVT VT =
N->getValueType(0);
17499 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(
LHS);
17500 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(
RHS);
17502 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
17503 LHS.getOperand(1).isUndef() &&
RHS.getOperand(1).isUndef() &&
17507 LHS.getOperand(0),
RHS.getOperand(0));
17521 if (
auto C = dyn_cast<ConstantSDNode>(
N->getOperand(2))) {
17522 int ShiftAmt =
C->getSExtValue();
17523 if (ShiftAmt == 0) {
17529 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17530 unsigned NewOpcode =
17546 unsigned IntNo =
N->getConstantOperandVal(0);
17557 case Intrinsic::arm_neon_vshifts:
17558 case Intrinsic::arm_neon_vshiftu:
17559 case Intrinsic::arm_neon_vrshifts:
17560 case Intrinsic::arm_neon_vrshiftu:
17561 case Intrinsic::arm_neon_vrshiftn:
17562 case Intrinsic::arm_neon_vqshifts:
17563 case Intrinsic::arm_neon_vqshiftu:
17564 case Intrinsic::arm_neon_vqshiftsu:
17565 case Intrinsic::arm_neon_vqshiftns:
17566 case Intrinsic::arm_neon_vqshiftnu:
17567 case Intrinsic::arm_neon_vqshiftnsu:
17568 case Intrinsic::arm_neon_vqrshiftns:
17569 case Intrinsic::arm_neon_vqrshiftnu:
17570 case Intrinsic::arm_neon_vqrshiftnsu: {
17571 EVT VT =
N->getOperand(1).getValueType();
17573 unsigned VShiftOpc = 0;
17576 case Intrinsic::arm_neon_vshifts:
17577 case Intrinsic::arm_neon_vshiftu:
17582 if (
isVShiftRImm(
N->getOperand(2), VT,
false,
true, Cnt)) {
17589 case Intrinsic::arm_neon_vrshifts:
17590 case Intrinsic::arm_neon_vrshiftu:
17595 case Intrinsic::arm_neon_vqshifts:
17596 case Intrinsic::arm_neon_vqshiftu:
17601 case Intrinsic::arm_neon_vqshiftsu:
17606 case Intrinsic::arm_neon_vrshiftn:
17607 case Intrinsic::arm_neon_vqshiftns:
17608 case Intrinsic::arm_neon_vqshiftnu:
17609 case Intrinsic::arm_neon_vqshiftnsu:
17610 case Intrinsic::arm_neon_vqrshiftns:
17611 case Intrinsic::arm_neon_vqrshiftnu:
17612 case Intrinsic::arm_neon_vqrshiftnsu:
17624 case Intrinsic::arm_neon_vshifts:
17625 case Intrinsic::arm_neon_vshiftu:
17628 case Intrinsic::arm_neon_vrshifts:
17631 case Intrinsic::arm_neon_vrshiftu:
17634 case Intrinsic::arm_neon_vrshiftn:
17637 case Intrinsic::arm_neon_vqshifts:
17640 case Intrinsic::arm_neon_vqshiftu:
17643 case Intrinsic::arm_neon_vqshiftsu:
17646 case Intrinsic::arm_neon_vqshiftns:
17649 case Intrinsic::arm_neon_vqshiftnu:
17652 case Intrinsic::arm_neon_vqshiftnsu:
17655 case Intrinsic::arm_neon_vqrshiftns:
17658 case Intrinsic::arm_neon_vqrshiftnu:
17661 case Intrinsic::arm_neon_vqrshiftnsu:
17667 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17668 N->getOperand(1), DAG.
getConstant(Cnt, dl, MVT::i32));
17671 case Intrinsic::arm_neon_vshiftins: {
17672 EVT VT =
N->getOperand(1).getValueType();
17674 unsigned VShiftOpc = 0;
17678 else if (
isVShiftRImm(
N->getOperand(3), VT,
false,
true, Cnt))
17685 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17686 N->getOperand(1),
N->getOperand(2),
17690 case Intrinsic::arm_neon_vqrshifts:
17691 case Intrinsic::arm_neon_vqrshiftu:
17695 case Intrinsic::arm_neon_vbsl: {
17698 N->getOperand(2),
N->getOperand(3));
17700 case Intrinsic::arm_mve_vqdmlah:
17701 case Intrinsic::arm_mve_vqdmlash:
17702 case Intrinsic::arm_mve_vqrdmlah:
17703 case Intrinsic::arm_mve_vqrdmlash:
17704 case Intrinsic::arm_mve_vmla_n_predicated:
17705 case Intrinsic::arm_mve_vmlas_n_predicated:
17706 case Intrinsic::arm_mve_vqdmlah_predicated:
17707 case Intrinsic::arm_mve_vqdmlash_predicated:
17708 case Intrinsic::arm_mve_vqrdmlah_predicated:
17709 case Intrinsic::arm_mve_vqrdmlash_predicated: {
17714 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
17721 case Intrinsic::arm_mve_minv:
17722 case Intrinsic::arm_mve_maxv:
17723 case Intrinsic::arm_mve_minav:
17724 case Intrinsic::arm_mve_maxav:
17725 case Intrinsic::arm_mve_minv_predicated:
17726 case Intrinsic::arm_mve_maxv_predicated:
17727 case Intrinsic::arm_mve_minav_predicated:
17728 case Intrinsic::arm_mve_maxav_predicated: {
17731 unsigned BitWidth =
N->getOperand(2)->getValueType(0).getScalarSizeInBits();
17738 case Intrinsic::arm_mve_addv: {
17741 bool Unsigned =
N->getConstantOperandVal(2);
17746 case Intrinsic::arm_mve_addlv:
17747 case Intrinsic::arm_mve_addlv_predicated: {
17750 bool Unsigned =
N->getConstantOperandVal(2);
17751 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
17756 for (
unsigned i = 1, e =
N->getNumOperands(); i < e; i++)
17779 EVT VT =
N->getValueType(0);
17781 if (ST->isThumb1Only() &&
N->getOpcode() ==
ISD::SHL && VT == MVT::i32 &&
17782 N->getOperand(0)->getOpcode() ==
ISD::AND &&
17783 N->getOperand(0)->hasOneUse()) {
17791 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17800 if (AndMask == 255 || AndMask == 65535)
17804 if (MaskedBits > ShiftAmt) {
17819 if (ST->hasMVEIntegerOps())
17824 switch (
N->getOpcode()) {
17837 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
17838 unsigned VShiftOpc =
17841 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
17857 if (!LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed() ||
17860 EVT FromVT = LD->getValueType(0);
17861 EVT ToVT =
N->getValueType(0);
17868 unsigned NumElements = 0;
17869 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
17871 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
17873 if (NumElements == 0 ||
17883 SDValue BasePtr = LD->getBasePtr();
17884 Align Alignment = LD->getOriginalAlign();
17905 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
17906 Alignment, MMOFlags, AAInfo);
17912 if (FromEltVT == MVT::f16) {
17915 for (
unsigned i = 0; i < Loads.
size(); i++) {
17941 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
17945 EVT VT =
N->getValueType(0);
17949 if (VT == MVT::i32 &&
17950 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
17952 isa<ConstantSDNode>(Lane)) {
17955 switch (
N->getOpcode()) {
17969 if (ST->hasMVEIntegerOps())
17978 if (ST->hasMVEFloatOps())
17989 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
17993 EVT VT =
Op.getValueType();
17996 if (VT != MVT::i32 ||
17998 !isa<ConstantSDNode>(
Op.getOperand(1)) ||
18009 APInt MaxC = Max.getConstantOperandAPInt(1);
18012 !(MinC + 1).isPowerOf2())
18030 EVT VT =
N->getValueType(0);
18033 if (VT == MVT::i32)
18036 if (!ST->hasMVEIntegerOps())
18042 if (VT != MVT::v4i32 && VT != MVT::v8i16)
18045 auto IsSignedSaturate = [&](
SDNode *Min,
SDNode *Max) {
18053 if (VT == MVT::v4i32)
18054 SaturateC =
APInt(32, (1 << 15) - 1,
true);
18056 SaturateC =
APInt(16, (1 << 7) - 1,
true);
18063 MaxC != ~SaturateC)
18068 if (IsSignedSaturate(
N, N0.
getNode())) {
18071 if (VT == MVT::v4i32) {
18072 HalfVT = MVT::v8i16;
18073 ExtVT = MVT::v4i16;
18075 HalfVT = MVT::v16i8;
18090 auto IsUnsignedSaturate = [&](
SDNode *Min) {
18096 if (VT == MVT::v4i32)
18097 SaturateC =
APInt(32, (1 << 16) - 1,
true);
18099 SaturateC =
APInt(16, (1 << 8) - 1,
true);
18108 if (IsUnsignedSaturate(
N)) {
18112 if (VT == MVT::v4i32) {
18113 HalfVT = MVT::v8i16;
18114 ExtConst = 0x0000FFFF;
18116 HalfVT = MVT::v16i8;
18138 const APInt *CV = &
C->getAPIntValue();
18156 SDValue Op0 = CMOV->getOperand(0);
18157 SDValue Op1 = CMOV->getOperand(1);
18158 auto CC = CMOV->getConstantOperandAPInt(2).getLimitedValue();
18159 SDValue CmpZ = CMOV->getOperand(3);
18195 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18202 if ((OrCI & Known.
Zero) != OrCI)
18208 EVT VT =
X.getValueType();
18209 unsigned BitInX = AndC->
logBase2();
18217 for (
unsigned BitInY = 0, NumActiveBits = OrCI.
getActiveBits();
18218 BitInY < NumActiveBits; ++BitInY) {
18219 if (OrCI[BitInY] == 0)
18222 Mask.setBit(BitInY);
18239 switch (
N->getOpcode()) {
18243 if (!isa<ConstantSDNode>(
N.getOperand(1)))
18245 if (!cast<ConstantSDNode>(
N.getOperand(1))->isOne())
18251 auto *Const = dyn_cast<ConstantSDNode>(
N.getOperand(1));
18254 if (Const->isZero())
18256 else if (Const->isOne())
18260 CC = cast<CondCodeSDNode>(
N.getOperand(2))->get();
18264 unsigned IntOp =
N.getConstantOperandVal(1);
18265 if (IntOp != Intrinsic::test_start_loop_iterations &&
18266 IntOp != Intrinsic::loop_decrement_reg)
18292 bool Negate =
false;
18298 Cond =
N->getOperand(1);
18299 Dest =
N->getOperand(2);
18302 CC = cast<CondCodeSDNode>(
N->getOperand(1))->get();
18303 Cond =
N->getOperand(2);
18304 Dest =
N->getOperand(4);
18305 if (
auto *Const = dyn_cast<ConstantSDNode>(
N->getOperand(3))) {
18306 if (!Const->isOne() && !Const->isZero())
18308 Imm = Const->getZExtValue();
18336 assert((IsTrueIfZero(
CC, Imm) || IsFalseIfZero(
CC, Imm)) &&
18337 "unsupported condition");
18342 unsigned IntOp =
Int->getConstantOperandVal(1);
18343 assert((
N->hasOneUse() &&
N->user_begin()->getOpcode() ==
ISD::BR) &&
18344 "expected single br user");
18345 SDNode *Br = *
N->user_begin();
18355 if (IntOp == Intrinsic::test_start_loop_iterations) {
18359 if (IsTrueIfZero(
CC, Imm)) {
18360 SDValue Ops[] = {Chain, Setup, Dest};
18365 UpdateUncondBr(Br, Dest, DAG);
18367 SDValue Ops[] = {Chain, Setup, OtherTarget};
18380 DAG.
getVTList(MVT::i32, MVT::Other), Args);
18388 if (
Target == OtherTarget)
18389 UpdateUncondBr(Br, Dest, DAG);
18425 LHS->getOperand(0)->getOperand(2),
18426 LHS->getOperand(0)->getOperand(3));
18440 EVT VT =
N->getValueType(0);
18444 SDValue FalseVal =
N->getOperand(0);
18445 SDValue TrueVal =
N->getOperand(1);
18450 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops()) {
18488 LHS->getOperand(2),
LHS->getOperand(3));
18498 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ ||
18502 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
18504 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
18513 if (!Subtarget->
isThumb1Only() && Subtarget->hasV5TOps()) {
18578 const APInt *TrueConst;
18580 ((FalseVal.getOpcode() ==
ARMISD::SUBC && FalseVal.getOperand(0) ==
LHS &&
18581 FalseVal.getOperand(1) ==
RHS) ||
18585 unsigned ShiftAmount = TrueConst->
logBase2();
18600 if (Known.
Zero == 0xfffffffe)
18603 else if (Known.
Zero == 0xffffff00)
18606 else if (Known.
Zero == 0xffff0000)
18619 EVT DstVT =
N->getValueType(0);
18622 if (ST->hasMVEIntegerOps() && Src.getOpcode() ==
ARMISD::VDUP) {
18623 EVT SrcVT = Src.getValueType();
18631 Src.getOperand(0).getValueType().getScalarSizeInBits() <=
18632 Src.getValueType().getScalarSizeInBits())
18633 Src = Src.getOperand(0);
18637 EVT SrcVT = Src.getValueType();
18657 EVT VT =
N->getValueType(0);
18665 if (
N->getNumOperands() == 2 &&
18669 N->getOperand(0).getOperand(1),
18670 N->getOperand(1).getOperand(0),
18671 N->getOperand(1).getOperand(1));
18674 if (
N->getNumOperands() == 2 &&
18677 auto *S0 = cast<ShuffleVectorSDNode>(
N->getOperand(0).getNode());
18678 auto *
S1 = cast<ShuffleVectorSDNode>(
N->getOperand(1).getNode());
18680 if (S0->getOperand(0) ==
S1->getOperand(0) &&
18681 S0->getOperand(1) ==
S1->getOperand(1)) {
18684 Mask.append(
S1->getMask().begin(),
S1->getMask().end());
18704 return Op.getOpcode() == ISD::BUILD_VECTOR ||
18705 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
18706 (Op.getOpcode() == ISD::BITCAST &&
18707 Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR);
18710 for (
unsigned Op = 0;
Op <
N->getNumOperands();
Op++) {
18712 for (
unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
18729 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18730 int NumIns =
N->getNumOperands();
18731 assert((NumIns == 2 || NumIns == 4) &&
18732 "Expected 2 or 4 inputs to an MVETrunc");
18734 if (
N->getNumOperands() == 4)
18738 for (
int I = 0;
I < NumIns;
I++) {
18740 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18760 if (!LD || !LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed())
18763 EVT FromVT = LD->getMemoryVT();
18764 EVT ToVT =
N->getValueType(0);
18771 unsigned NumElements = 0;
18772 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
18774 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
18776 assert(NumElements != 0);
18782 LD->getExtensionType() != NewExtType)
18789 SDValue BasePtr = LD->getBasePtr();
18790 Align Alignment = LD->getOriginalAlign();
18809 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
18810 Alignment, MMOFlags, AAInfo);
18826 EVT VT =
N->getValueType(0);
18828 assert(
N->getNumValues() == 2 &&
"Expected MVEEXT with 2 elements");
18829 assert((VT == MVT::v4i32 || VT == MVT::v8i16) &&
"Unexpected MVEEXT type");
18831 EVT ExtVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18833 auto Extend = [&](
SDValue V) {
18843 SDValue Ext = Extend(
N->getOperand(0));
18848 if (
auto *SVN = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(0))) {
18851 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
18856 auto CheckInregMask = [&](
int Start,
int Offset) {
18864 if (CheckInregMask(0, 0))
18866 else if (CheckInregMask(0, 1))
18867 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18868 else if (CheckInregMask(0, Mask.size()))
18870 else if (CheckInregMask(0, Mask.size() + 1))
18871 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18876 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18880 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18887 if (
N->getOperand(0)->getOpcode() ==
ISD::LOAD)
18897 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18898 int NumOuts =
N->getNumValues();
18899 assert((NumOuts == 2 || NumOuts == 4) &&
18900 "Expected 2 or 4 outputs to an MVEEXT");
18901 EVT LoadVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18903 if (
N->getNumOperands() == 4)
18909 StackPtr, MPI,
Align(4));
18912 for (
int I = 0;
I < NumOuts;
I++) {
18914 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18915 DAG.
getConstant(
I * 16 / NumOuts,
DL, StackPtr.getValueType()));
18920 VT, Chain,
Ptr, MPI, LoadVT,
Align(4));
18929 switch (
N->getOpcode()) {
19046 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19053 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19064 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19072 unsigned LowWidth =
N->getOperand(0).getValueType().getSizeInBits();
19074 unsigned HighWidth =
N->getOperand(1).getValueType().getSizeInBits();
19082 unsigned HighWidth =
N->getOperand(0).getValueType().getSizeInBits();
19084 unsigned LowWidth =
N->getOperand(1).getValueType().getSizeInBits();
19092 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19103 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19111 if (
N->getOperand(1) ==
N->getOperand(2))
19112 return N->getOperand(1);
19116 switch (
N->getConstantOperandVal(1)) {
19117 case Intrinsic::arm_neon_vld1:
19118 case Intrinsic::arm_neon_vld1x2:
19119 case Intrinsic::arm_neon_vld1x3:
19120 case Intrinsic::arm_neon_vld1x4:
19121 case Intrinsic::arm_neon_vld2:
19122 case Intrinsic::arm_neon_vld3:
19123 case Intrinsic::arm_neon_vld4:
19124 case Intrinsic::arm_neon_vld2lane:
19125 case Intrinsic::arm_neon_vld3lane:
19126 case Intrinsic::arm_neon_vld4lane:
19127 case Intrinsic::arm_neon_vld2dup:
19128 case Intrinsic::arm_neon_vld3dup:
19129 case Intrinsic::arm_neon_vld4dup:
19130 case Intrinsic::arm_neon_vst1:
19131 case Intrinsic::arm_neon_vst1x2:
19132 case Intrinsic::arm_neon_vst1x3:
19133 case Intrinsic::arm_neon_vst1x4:
19134 case Intrinsic::arm_neon_vst2:
19135 case Intrinsic::arm_neon_vst3:
19136 case Intrinsic::arm_neon_vst4:
19137 case Intrinsic::arm_neon_vst2lane:
19138 case Intrinsic::arm_neon_vst3lane:
19139 case Intrinsic::arm_neon_vst4lane:
19141 case Intrinsic::arm_mve_vld2q:
19142 case Intrinsic::arm_mve_vld4q:
19143 case Intrinsic::arm_mve_vst2q:
19144 case Intrinsic::arm_mve_vst4q:
19161 unsigned *
Fast)
const {
19170 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19172 if (AllowsUnaligned) {
19174 *
Fast = Subtarget->hasV7Ops();
19179 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19183 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->
isLittle())) {
19190 if (!Subtarget->hasMVEIntegerOps())
19194 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19195 Ty == MVT::v2i1)) {
19203 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19219 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19220 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19221 Ty == MVT::v2f64) {
19234 if ((
Op.isMemcpy() ||
Op.isZeroMemset()) && Subtarget->hasNEON() &&
19235 !FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat)) {
19237 if (
Op.size() >= 16 &&
19243 }
else if (
Op.size() >= 8 &&
19264 return (SrcBits == 64 && DestBits == 32);
19273 return (SrcBits == 64 && DestBits == 32);
19309 return Subtarget->hasFullFP16();
19316 if (!Subtarget->hasMVEIntegerOps())
19334 if (
auto *Ld = dyn_cast<MaskedLoadSDNode>(ExtVal.
getOperand(0))) {
19335 if (Ld->isExpandingLoad())
19339 if (Subtarget->hasMVEIntegerOps())
19384bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(
const MachineFunction &MF,
19386 if (Subtarget->useSoftFloat())
19395 return Subtarget->hasMVEFloatOps();
19413 unsigned Scale = 1;
19430 if ((V & (Scale - 1)) != 0)
19432 return isUInt<5>(V / Scale);
19439 if (VT.
isVector() && Subtarget->hasNEON())
19442 !Subtarget->hasMVEFloatOps())
19445 bool IsNeg =
false;
19451 unsigned NumBytes = std::max((
unsigned)VT.
getSizeInBits() / 8, 1U);
19454 if (VT.
isVector() && Subtarget->hasMVEIntegerOps()) {
19458 return isShiftedUInt<7,2>(V);
19461 return isShiftedUInt<7,1>(V);
19463 return isUInt<7>(V);
19470 if (VT.
isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19471 return isShiftedUInt<8, 1>(V);
19474 return isShiftedUInt<8, 2>(V);
19476 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19479 return isUInt<8>(V);
19480 return isUInt<12>(V);
19506 default:
return false;
19511 return isUInt<12>(V);
19514 return isUInt<8>(V);
19519 return isShiftedUInt<8, 2>(V);
19525 int Scale = AM.
Scale;
19530 default:
return false;
19538 Scale = Scale & ~1;
19539 return Scale == 2 || Scale == 4 || Scale == 8;
19556 if (Scale & 1)
return false;
19563 const int Scale = AM.
Scale;
19573 return (Scale == 1) || (!AM.
HasBaseReg && Scale == 2);
19589 switch (AM.
Scale) {
19606 int Scale = AM.
Scale;
19608 default:
return false;
19612 if (Scale < 0) Scale = -Scale;
19620 if (Scale == 1 || (AM.
HasBaseReg && Scale == -1))
19633 if (Scale & 1)
return false;
19646 if (!Subtarget->isThumb())
19653 return Imm >= 0 && Imm <= 255;
19662 int64_t AbsImm = std::abs(Imm);
19663 if (!Subtarget->isThumb())
19668 return AbsImm >= 0 && AbsImm <= 255;
19703 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
19707 int RHSC = (int)
RHS->getZExtValue();
19708 if (RHSC < 0 && RHSC > -256) {
19718 }
else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
19721 int RHSC = (int)
RHS->getZExtValue();
19722 if (RHSC < 0 && RHSC > -0x1000) {
19764 int RHSC = (int)
RHS->getZExtValue();
19765 if (RHSC < 0 && RHSC > -0x100) {
19770 }
else if (RHSC > 0 && RHSC < 0x100) {
19781 bool isSEXTLoad,
bool IsMasked,
bool isLE,
19786 if (!isa<ConstantSDNode>(
Ptr->getOperand(1)))
19792 bool CanChangeType = isLE && !IsMasked;
19795 int RHSC = (int)
RHS->getZExtValue();
19797 auto IsInRange = [&](
int RHSC,
int Limit,
int Scale) {
19798 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
19803 }
else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
19814 if (VT == MVT::v4i16) {
19815 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
19817 }
else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
19818 if (IsInRange(RHSC, 0x80, 1))
19820 }
else if (Alignment >= 4 &&
19821 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
19822 IsInRange(RHSC, 0x80, 4))
19824 else if (Alignment >= 2 &&
19825 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
19826 IsInRange(RHSC, 0x80, 2))
19828 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
19847 bool isSEXTLoad =
false;
19848 bool IsMasked =
false;
19850 Ptr = LD->getBasePtr();
19851 VT = LD->getMemoryVT();
19852 Alignment = LD->getAlign();
19854 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
19855 Ptr = ST->getBasePtr();
19856 VT = ST->getMemoryVT();
19857 Alignment = ST->getAlign();
19859 Ptr = LD->getBasePtr();
19860 VT = LD->getMemoryVT();
19861 Alignment = LD->getAlign();
19865 Ptr = ST->getBasePtr();
19866 VT = ST->getMemoryVT();
19867 Alignment = ST->getAlign();
19873 bool isLegal =
false;
19875 isLegal = Subtarget->hasMVEIntegerOps() &&
19877 Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
19905 bool isSEXTLoad =
false, isNonExt;
19906 bool IsMasked =
false;
19908 VT = LD->getMemoryVT();
19909 Ptr = LD->getBasePtr();
19910 Alignment = LD->getAlign();
19913 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
19914 VT = ST->getMemoryVT();
19915 Ptr = ST->getBasePtr();
19916 Alignment = ST->getAlign();
19917 isNonExt = !ST->isTruncatingStore();
19919 VT = LD->getMemoryVT();
19920 Ptr = LD->getBasePtr();
19921 Alignment = LD->getAlign();
19926 VT = ST->getMemoryVT();
19927 Ptr = ST->getBasePtr();
19928 Alignment = ST->getAlign();
19929 isNonExt = !ST->isTruncatingStore();
19937 assert(
Op->getValueType(0) == MVT::i32 &&
"Non-i32 post-inc op?!");
19938 if (
Op->getOpcode() !=
ISD::ADD || !isNonExt)
19940 auto *
RHS = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
19941 if (!
RHS ||
RHS->getZExtValue() != 4)
19943 if (Alignment <
Align(4))
19947 Base =
Op->getOperand(0);
19953 bool isLegal =
false;
19955 isLegal = Subtarget->hasMVEIntegerOps() &&
19988 const APInt &DemandedElts,
19990 unsigned Depth)
const {
19993 switch (
Op.getOpcode()) {
20000 if (
Op.getResNo() == 0) {
20026 case Intrinsic::arm_ldaex:
20027 case Intrinsic::arm_ldrex: {
20028 EVT VT = cast<MemIntrinsicSDNode>(
Op)->getMemoryVT();
20042 const APInt &Mask =
Op.getConstantOperandAPInt(2);
20043 Known.
Zero &= Mask;
20049 const SDValue &SrcSV =
Op.getOperand(0);
20055 "VGETLANE index out of bounds");
20060 EVT VT =
Op.getValueType();
20067 Known = Known.
sext(DstSz);
20069 Known = Known.
zext(DstSz);
20077 Known = KnownOp.
zext(32);
20117 EVT VT =
Op.getValueType();
20123 assert(VT == MVT::i32 &&
"Unexpected integer type");
20130 unsigned Mask =
C->getZExtValue();
20133 unsigned ShrunkMask = Mask & Demanded;
20134 unsigned ExpandedMask = Mask | ~Demanded;
20138 if (ShrunkMask == 0)
20144 if (ExpandedMask == ~0U)
20147 auto IsLegalMask = [ShrunkMask, ExpandedMask](
unsigned Mask) ->
bool {
20148 return (ShrunkMask & Mask) == ShrunkMask && (~ExpandedMask & Mask) == 0;
20150 auto UseMask = [Mask,
Op, VT, &TLO](
unsigned NewMask) ->
bool {
20151 if (NewMask == Mask)
20160 if (IsLegalMask(0xFF))
20161 return UseMask(0xFF);
20164 if (IsLegalMask(0xFFFF))
20165 return UseMask(0xFFFF);
20169 if (ShrunkMask < 256)
20170 return UseMask(ShrunkMask);
20174 if ((
int)ExpandedMask <= -2 && (
int)ExpandedMask >= -256)
20175 return UseMask(ExpandedMask);
20190 unsigned Depth)
const {
20191 unsigned Opc =
Op.getOpcode();
20199 if (
Op.getResNo() == 0 && !
Op->hasAnyUseOfValue(1) &&
20200 isa<ConstantSDNode>(
Op->getOperand(2))) {
20201 unsigned ShAmt =
Op->getConstantOperandVal(2);
20213 unsigned ModImm =
Op.getConstantOperandVal(1);
20214 unsigned EltBits = 0;
20216 if ((OriginalDemandedBits & Mask) == 0)
20222 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO,
Depth);
20231 if (!Subtarget->hasV6Ops())
20237 SplitString(AsmStr, AsmPieces,
";\n");
20239 switch (AsmPieces.
size()) {
20240 default:
return false;
20242 AsmStr = AsmPieces[0];
20244 SplitString(AsmStr, AsmPieces,
" \t,");
20247 if (AsmPieces.
size() == 3 &&
20248 AsmPieces[0] ==
"rev" && AsmPieces[1] ==
"$0" && AsmPieces[2] ==
"$1" &&
20249 IA->getConstraintString().compare(0, 4,
"=l,l") == 0) {
20272 if (ConstraintVT.
isVector() && Subtarget->hasNEON() &&
20284 unsigned S = Constraint.
size();
20286 switch (Constraint[0]) {
20298 }
else if (S == 2) {
20299 switch (Constraint[0]) {
20316 Value *CallOperandVal =
info.CallOperandVal;
20319 if (!CallOperandVal)
20323 switch (*constraint) {
20329 if (Subtarget->isThumb())
20343using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20347 switch (Constraint.
size()) {
20350 switch (Constraint[0]) {
20352 if (Subtarget->isThumb())
20353 return RCPair(0U, &ARM::tGPRRegClass);
20354 return RCPair(0U, &ARM::GPRRegClass);
20356 if (Subtarget->isThumb())
20357 return RCPair(0U, &ARM::hGPRRegClass);
20361 return RCPair(0U, &ARM::tGPRRegClass);
20362 return RCPair(0U, &ARM::GPRRegClass);
20364 if (VT == MVT::Other)
20366 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20367 return RCPair(0U, &ARM::SPRRegClass);
20369 return RCPair(0U, &ARM::DPRRegClass);
20371 return RCPair(0U, &ARM::QPRRegClass);
20374 if (VT == MVT::Other)
20376 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20377 return RCPair(0U, &ARM::SPR_8RegClass);
20379 return RCPair(0U, &ARM::DPR_8RegClass);
20381 return RCPair(0U, &ARM::QPR_8RegClass);
20384 if (VT == MVT::Other)
20386 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20387 return RCPair(0U, &ARM::SPRRegClass);
20389 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20391 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20397 if (Constraint[0] ==
'T') {
20398 switch (Constraint[1]) {
20402 return RCPair(0U, &ARM::tGPREvenRegClass);
20404 return RCPair(0U, &ARM::tGPROddRegClass);
20413 if (
StringRef(
"{cc}").equals_insensitive(Constraint))
20414 return std::make_pair(
unsigned(ARM::CPSR), &ARM::CCRRegClass);
20423 std::vector<SDValue> &Ops,
20428 if (Constraint.
size() != 1)
20431 char ConstraintLetter = Constraint[0];
20432 switch (ConstraintLetter) {
20435 case 'I':
case 'J':
case 'K':
case 'L':
20436 case 'M':
case 'N':
case 'O':
20441 int64_t CVal64 =
C->getSExtValue();
20442 int CVal = (int) CVal64;
20445 if (CVal != CVal64)
20448 switch (ConstraintLetter) {
20452 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
20453 if (CVal >= 0 && CVal <= 65535)
20460 if (CVal >= 0 && CVal <= 255)
20462 }
else if (Subtarget->
isThumb2()) {
20481 if (CVal >= -255 && CVal <= -1)
20487 if (CVal >= -4095 && CVal <= 4095)
20500 }
else if (Subtarget->
isThumb2()) {
20523 if (CVal >= -7 && CVal < 7)
20525 }
else if (Subtarget->
isThumb2()) {
20548 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
20554 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
20562 if (CVal >= 0 && CVal <= 31)
20571 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
20580 if (Result.getNode()) {
20581 Ops.push_back(Result);
20591 "Unhandled Opcode in getDivRemLibcall");
20597 case MVT::i8: LC =
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
20598 case MVT::i16: LC =
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
20599 case MVT::i32: LC =
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
20600 case MVT::i64: LC =
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
20609 "Unhandled Opcode in getDivRemArgList");
20614 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
20615 EVT ArgVT =
N->getOperand(i).getValueType();
20617 Entry.Node =
N->getOperand(i);
20621 Args.push_back(Entry);
20632 "Register-based DivRem lowering only");
20633 unsigned Opcode =
Op->getOpcode();
20635 "Invalid opcode for Div/Rem lowering");
20637 EVT VT =
Op->getValueType(0);
20640 if (VT == MVT::i64 && isa<ConstantSDNode>(
Op.getOperand(1))) {
20659 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
20660 : Subtarget->hasDivideInARMMode();
20661 if (hasDivide &&
Op->getValueType(0).isSimple() &&
20662 Op->getSimpleValueType(0) == MVT::i32) {
20664 const SDValue Dividend =
Op->getOperand(0);
20665 const SDValue Divisor =
Op->getOperand(1);
20666 SDValue Div = DAG.
getNode(DivOpcode, dl, VT, Dividend, Divisor);
20670 SDValue Values[2] = {Div, Rem};
20702 EVT VT =
N->getValueType(0);
20704 if (VT == MVT::i64 && isa<ConstantSDNode>(
N->getOperand(1))) {
20708 Result[0], Result[1]);
20712 std::vector<Type*> RetTyParams;
20713 Type *RetTyElement;
20723 RetTyParams.push_back(RetTyElement);
20724 RetTyParams.push_back(RetTyElement);
20741 CallLoweringInfo CLI(DAG);
20745 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
20748 SDNode *ResNode = CallResult.first.getNode();
20763 "no-stack-arg-probe")) {
20765 cast<ConstantSDNode>(
Op.getOperand(2))->getMaybeAlignValue();
20774 SDValue Ops[2] = { SP, Chain };
20791 SDValue Ops[2] = { NewSP, Chain };
20796 bool IsStrict =
Op->isStrictFPOpcode();
20797 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20798 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20800 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
20801 "Unexpected type for custom-lowering FP_EXTEND");
20804 "With both FP DP and 16, any FP conversion is legal!");
20806 assert(!(DstSz == 32 && Subtarget->hasFP16()) &&
20807 "With FP16, 16 to 32 conversion is legal!");
20810 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) {
20815 Loc,
Op.getValueType(), SrcVal);
20828 MakeLibCallOptions CallOptions;
20830 for (
unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
20831 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
20832 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
20833 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
20837 {DstVT, MVT::Other}, {Chain, SrcVal});
20844 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20845 "Unexpected type for custom-lowering FP_EXTEND");
20846 std::tie(SrcVal, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20851 return IsStrict ? DAG.
getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
20855 bool IsStrict =
Op->isStrictFPOpcode();
20857 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20859 EVT DstVT =
Op.getValueType();
20860 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20863 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
20864 "Unexpected type for custom-lowering FP_ROUND");
20867 "With both FP DP and 16, any FP conversion is legal!");
20872 if (SrcSz == 32 && Subtarget->hasFP16())
20877 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20878 "Unexpected type for custom-lowering FP_ROUND");
20879 MakeLibCallOptions CallOptions;
20882 std::tie(Result, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20894 if (v == 0xffffffff)
20906 bool ForCodeSize)
const {
20909 if (VT == MVT::f16 && Subtarget->hasFullFP16())
20911 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
20914 if (VT == MVT::f32)
20916 if (VT == MVT::f64 && Subtarget->hasFP64())
20927 unsigned Intrinsic)
const {
20928 switch (Intrinsic) {
20929 case Intrinsic::arm_neon_vld1:
20930 case Intrinsic::arm_neon_vld2:
20931 case Intrinsic::arm_neon_vld3:
20932 case Intrinsic::arm_neon_vld4:
20933 case Intrinsic::arm_neon_vld2lane:
20934 case Intrinsic::arm_neon_vld3lane:
20935 case Intrinsic::arm_neon_vld4lane:
20936 case Intrinsic::arm_neon_vld2dup:
20937 case Intrinsic::arm_neon_vld3dup:
20938 case Intrinsic::arm_neon_vld4dup: {
20941 auto &
DL =
I.getDataLayout();
20942 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
20944 Info.ptrVal =
I.getArgOperand(0);
20946 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
20947 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
20952 case Intrinsic::arm_neon_vld1x2:
20953 case Intrinsic::arm_neon_vld1x3:
20954 case Intrinsic::arm_neon_vld1x4: {
20957 auto &
DL =
I.getDataLayout();
20958 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
20960 Info.ptrVal =
I.getArgOperand(
I.arg_size() - 1);
20962 Info.align =
I.getParamAlign(
I.arg_size() - 1).valueOrOne();
20967 case Intrinsic::arm_neon_vst1:
20968 case Intrinsic::arm_neon_vst2:
20969 case Intrinsic::arm_neon_vst3:
20970 case Intrinsic::arm_neon_vst4:
20971 case Intrinsic::arm_neon_vst2lane:
20972 case Intrinsic::arm_neon_vst3lane:
20973 case Intrinsic::arm_neon_vst4lane: {
20976 auto &
DL =
I.getDataLayout();
20977 unsigned NumElts = 0;
20978 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
20979 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
20982 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
20985 Info.ptrVal =
I.getArgOperand(0);
20987 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
20988 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
20993 case Intrinsic::arm_neon_vst1x2:
20994 case Intrinsic::arm_neon_vst1x3:
20995 case Intrinsic::arm_neon_vst1x4: {
20998 auto &
DL =
I.getDataLayout();
20999 unsigned NumElts = 0;
21000 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21001 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21004 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21007 Info.ptrVal =
I.getArgOperand(0);
21009 Info.align =
I.getParamAlign(0).valueOrOne();
21014 case Intrinsic::arm_mve_vld2q:
21015 case Intrinsic::arm_mve_vld4q: {
21018 Type *VecTy = cast<StructType>(
I.getType())->getElementType(1);
21019 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21021 Info.ptrVal =
I.getArgOperand(0);
21028 case Intrinsic::arm_mve_vst2q:
21029 case Intrinsic::arm_mve_vst4q: {
21032 Type *VecTy =
I.getArgOperand(1)->getType();
21033 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21035 Info.ptrVal =
I.getArgOperand(0);
21042 case Intrinsic::arm_mve_vldr_gather_base:
21043 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21045 Info.ptrVal =
nullptr;
21051 case Intrinsic::arm_mve_vldr_gather_base_wb:
21052 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21054 Info.ptrVal =
nullptr;
21060 case Intrinsic::arm_mve_vldr_gather_offset:
21061 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21063 Info.ptrVal =
nullptr;
21065 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
21072 case Intrinsic::arm_mve_vstr_scatter_base:
21073 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21075 Info.ptrVal =
nullptr;
21081 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21082 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21084 Info.ptrVal =
nullptr;
21090 case Intrinsic::arm_mve_vstr_scatter_offset:
21091 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21093 Info.ptrVal =
nullptr;
21095 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(3))->getZExtValue();
21102 case Intrinsic::arm_ldaex:
21103 case Intrinsic::arm_ldrex: {
21104 auto &
DL =
I.getDataLayout();
21105 Type *ValTy =
I.getParamElementType(0);
21108 Info.ptrVal =
I.getArgOperand(0);
21110 Info.align =
DL.getABITypeAlign(ValTy);
21114 case Intrinsic::arm_stlex:
21115 case Intrinsic::arm_strex: {
21116 auto &
DL =
I.getDataLayout();
21117 Type *ValTy =
I.getParamElementType(1);
21120 Info.ptrVal =
I.getArgOperand(1);
21122 Info.align =
DL.getABITypeAlign(ValTy);
21126 case Intrinsic::arm_stlexd:
21127 case Intrinsic::arm_strexd:
21129 Info.memVT = MVT::i64;
21130 Info.ptrVal =
I.getArgOperand(2);
21136 case Intrinsic::arm_ldaexd:
21137 case Intrinsic::arm_ldrexd:
21139 Info.memVT = MVT::i64;
21140 Info.ptrVal =
I.getArgOperand(0);
21160 if (Bits == 0 || Bits > 32)
21166 unsigned Index)
const {
21176 if (!Subtarget->hasDataBarrier()) {
21180 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21215 if (Subtarget->preferISHSTBarriers())
21248 bool has64BitAtomicStore;
21250 has64BitAtomicStore =
false;
21251 else if (Subtarget->isThumb())
21252 has64BitAtomicStore = Subtarget->hasV7Ops();
21254 has64BitAtomicStore = Subtarget->hasV6Ops();
21256 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21270 bool has64BitAtomicLoad;
21272 has64BitAtomicLoad =
false;
21273 else if (Subtarget->isThumb())
21274 has64BitAtomicLoad = Subtarget->hasV7Ops();
21276 has64BitAtomicLoad = Subtarget->hasV6Ops();
21293 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21294 else if (Subtarget->isThumb())
21295 hasAtomicRMW = Subtarget->hasV7Ops();
21297 hasAtomicRMW = Subtarget->hasV6Ops();
21298 if (
Size <= (Subtarget->
isMClass() ? 32U : 64U) && hasAtomicRMW) {
21321 bool HasAtomicCmpXchg;
21323 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21324 else if (Subtarget->isThumb())
21325 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21327 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21329 HasAtomicCmpXchg &&
Size <= (Subtarget->
isMClass() ? 32U : 64U))
21336 return InsertFencesForAtomic;
21349 M.getOrInsertGlobal(
"__security_cookie",
21357 F->addParamAttr(0, Attribute::AttrKind::InReg);
21363 return M.getGlobalVariable(
"__security_cookie");
21370 return M.getFunction(
"__security_check_cookie");
21375 unsigned &
Cost)
const {
21377 if (!Subtarget->hasNEON())
21389 if (!isa<ConstantInt>(
Idx))
21404 return Subtarget->hasV6T2Ops();
21408 return Subtarget->hasV6T2Ops();
21413 if (!Subtarget->hasV7Ops())
21419 if (!Mask || Mask->getValue().getBitWidth() > 32u)
21421 auto MaskVal =
unsigned(Mask->getValue().getZExtValue());
21446 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
21458 Lo, Builder.
CreateShl(
Hi, ConstantInt::get(ValueTy, 32)),
"val64");
21461 Type *Tys[] = {
Addr->getType() };
21462 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
21466 0,
Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
21472 if (!Subtarget->hasV7Ops())
21488 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
21498 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
21499 Type *Tys[] = {
Addr->getType() };
21521 return (
DL.getTypeSizeInBits(VecTy) + 127) / 128;
21528 unsigned VecSize =
DL.getTypeSizeInBits(VecTy);
21531 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
21539 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
21547 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
21550 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
21555 if (Subtarget->hasNEON() && VecSize == 64)
21557 return VecSize % 128 == 0;
21561 if (Subtarget->hasNEON())
21563 if (Subtarget->hasMVEIntegerOps())
21583 "Invalid interleave factor");
21584 assert(!Shuffles.
empty() &&
"Empty shufflevector input");
21586 "Unmatched number of shufflevectors and indices");
21588 auto *VecTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21589 Type *EltTy = VecTy->getElementType();
21612 if (NumLoads > 1) {
21616 VecTy->getNumElements() / NumLoads);
21621 auto createLoadIntrinsic = [&](
Value *BaseAddr) {
21622 if (Subtarget->hasNEON()) {
21624 Type *Tys[] = {VecTy, PtrTy};
21625 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
21626 Intrinsic::arm_neon_vld3,
21627 Intrinsic::arm_neon_vld4};
21636 assert((Factor == 2 || Factor == 4) &&
21637 "expected interleave factor of 2 or 4 for MVE");
21639 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
21641 Type *Tys[] = {VecTy, PtrTy};
21655 for (
unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
21660 VecTy->getNumElements() * Factor);
21662 CallInst *VldN = createLoadIntrinsic(BaseAddr);
21666 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21668 unsigned Index = Indices[i];
21678 SubVecs[SV].push_back(SubVec);
21687 auto &SubVec = SubVecs[SVI];
21690 SVI->replaceAllUsesWith(WideVec);
21724 unsigned Factor)
const {
21726 "Invalid interleave factor");
21728 auto *VecTy = cast<FixedVectorType>(SVI->
getType());
21729 assert(VecTy->getNumElements() % Factor == 0 &&
"Invalid interleaved store");
21731 unsigned LaneLen = VecTy->getNumElements() / Factor;
21732 Type *EltTy = VecTy->getElementType();
21736 Align Alignment = SI->getAlign();
21753 Type *IntTy =
DL.getIntPtrType(EltTy);
21765 Value *BaseAddr = SI->getPointerOperand();
21767 if (NumStores > 1) {
21770 LaneLen /= NumStores;
21778 auto createStoreIntrinsic = [&](
Value *BaseAddr,
21780 if (Subtarget->hasNEON()) {
21781 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
21782 Intrinsic::arm_neon_vst3,
21783 Intrinsic::arm_neon_vst4};
21784 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21785 Type *Tys[] = {PtrTy, SubVecTy};
21793 assert((Factor == 2 || Factor == 4) &&
21794 "expected interleave factor of 2 or 4 for MVE");
21796 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
21797 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21798 Type *Tys[] = {PtrTy, SubVecTy};
21803 for (
unsigned F = 0;
F < Factor;
F++) {
21811 for (
unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
21814 if (StoreCount > 0)
21816 BaseAddr, LaneLen * Factor);
21821 for (
unsigned i = 0; i < Factor; i++) {
21822 unsigned IdxI = StoreCount * LaneLen * Factor + i;
21823 if (Mask[IdxI] >= 0) {
21827 unsigned StartMask = 0;
21828 for (
unsigned j = 1; j < LaneLen; j++) {
21829 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
21830 if (Mask[IdxJ * Factor + IdxI] >= 0) {
21831 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
21846 createStoreIntrinsic(BaseAddr, Shuffles);
21861 if (
auto *ST = dyn_cast<StructType>(Ty)) {
21862 for (
unsigned i = 0; i < ST->getNumElements(); ++i) {
21866 Members += SubMembers;
21868 }
else if (
auto *AT = dyn_cast<ArrayType>(Ty)) {
21872 Members += SubMembers * AT->getNumElements();
21883 }
else if (
auto *VT = dyn_cast<VectorType>(Ty)) {
21890 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
21892 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
21894 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
21907 return (Members > 0 && Members <= 4);
21913 const Align ABITypeAlign =
DL.getABITypeAlign(ArgTy);
21915 return ABITypeAlign;
21920 assert(StackAlign &&
"data layout string is missing stack alignment");
21921 return std::min(ABITypeAlign, *StackAlign);
21930 if (getEffectiveCallingConv(CallConv, isVarArg) !=
21940 return IsHA || IsIntArray;
21944 const Constant *PersonalityFn)
const {
21951 const Constant *PersonalityFn)
const {
21963void ARMTargetLowering::insertCopiesSplitCSR(
21967 const MCPhysReg *IStart =
TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
21977 RC = &ARM::GPRRegClass;
21978 else if (ARM::DPRRegClass.
contains(*
I))
21979 RC = &ARM::DPRRegClass;
21989 assert(Entry->getParent()->getFunction().hasFnAttribute(
21990 Attribute::NoUnwind) &&
21991 "Function should be nounwind in insertCopiesSplitCSR!");
21992 Entry->addLiveIn(*
I);
21997 for (
auto *Exit : Exits)
21999 TII->get(TargetOpcode::COPY), *
I)
22010 return Subtarget->hasMVEIntegerOps();
22015 auto *VTy = dyn_cast<FixedVectorType>(Ty);
22020 unsigned NumElements = VTy->getNumElements();
22027 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22028 return Subtarget->hasMVEFloatOps();
22033 return Subtarget->hasMVEIntegerOps() &&
22034 (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
22035 ScalarTy->isIntegerTy(32));
22047 assert(TyWidth >= 128 &&
"Width of vector type must be at least 128 bits");
22049 if (TyWidth > 128) {
22054 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22056 auto *LowerSplitA =
B.CreateShuffleVector(InputA, LowerSplitMask);
22057 auto *LowerSplitB =
B.CreateShuffleVector(InputB, LowerSplitMask);
22058 auto *UpperSplitA =
B.CreateShuffleVector(InputA, UpperSplitMask);
22059 auto *UpperSplitB =
B.CreateShuffleVector(InputB, UpperSplitMask);
22060 Value *LowerSplitAcc =
nullptr;
22061 Value *UpperSplitAcc =
nullptr;
22064 LowerSplitAcc =
B.CreateShuffleVector(
Accumulator, LowerSplitMask);
22065 UpperSplitAcc =
B.CreateShuffleVector(
Accumulator, UpperSplitMask);
22069 B, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
22071 B, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);
22074 return B.CreateShuffleVector(LowerSplitInt, UpperSplitInt, JoinMask);
22081 ConstRotation = ConstantInt::get(IntTy, (
int)Rotation);
22084 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmlaq, Ty,
22086 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmulq, Ty,
22087 {ConstRotation, InputB, InputA});
22092 auto *ConstHalving = ConstantInt::get(IntTy, 1);
22095 ConstRotation = ConstantInt::get(IntTy, 0);
22097 ConstRotation = ConstantInt::get(IntTy, 1);
22099 if (!ConstRotation)
22102 return B.CreateIntrinsic(Intrinsic::arm_mve_vcaddq, Ty,
22103 {ConstHalving, ConstRotation, InputA, InputB});
unsigned const MachineRegisterInfo * MRI
static bool isAddSubSExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG, bool isSigned)
static bool isZeroExtended(SDValue N, SelectionDAG &DAG)
static const MCPhysReg GPRArgRegs[]
static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1, SDValue V2, unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
static bool isSignExtended(SDValue N, SelectionDAG &DAG)
static bool isAddSubZExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG)
static bool isStore(int Opcode)
static bool isThumb(const MCSubtargetInfo &STI)
static SDValue LowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG)
static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total value size to 64 bits.
static cl::opt< unsigned > ConstpoolPromotionMaxSize("arm-promote-constant-max-size", cl::Hidden, cl::desc("Maximum size of constant to promote into a constant pool"), cl::init(64))
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isVTBLMask(ArrayRef< int > M, EVT VT)
static SDValue PerformSUBCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
static cl::opt< bool > EnableConstpoolPromotion("arm-promote-constant", cl::Hidden, cl::desc("Enable / disable promotion of unnamed_addr constants into " "constant pools"), cl::init(false))
static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG)
static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
static SDValue PerformExtractEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static const APInt * isPowerOf2Constant(SDValue V)
static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG)
static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static bool isValidMVECond(unsigned CC, bool IsFloat)
static SDValue PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC)
IntCCToARMCC - Convert a DAG integer condition code to an ARM CC.
static SDValue PerformSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSTORECombine - Target-specific dag combine xforms for ISD::STORE.
static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry, SelectionDAG &DAG)
static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isGTorGE(ISD::CondCode CC)
static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a vldN-lane (N > 1) intrinsic,...
static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask)
static bool isReverseMask(ArrayRef< int > M, EVT VT)
static bool isVZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of "vector_shuffle v,...
static SDValue PerformSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVMulVCTPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVMulVCTPCombine - VCVT (fixed-point to floating-point, Advanced SIMD) can replace combinations...
static SDValue createGPRPairNode2xi32(SelectionDAG &DAG, SDValue V0, SDValue V1)
static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG)
static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc)
static bool isVTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool CanInvertMVEVCMP(SDValue N)
static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG)
static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformShiftCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
PerformShiftCombine - Checks for immediate versions of vector shifts and lowers them.
static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, ARMCC::CondCodes &CondCode2)
FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static EVT getVectorTyFromPredicateVector(EVT VT)
static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG)
static SDValue handleCMSEValue(const SDValue &Value, const ISD::InputArg &Arg, SelectionDAG &DAG, const SDLoc &DL)
static SDValue PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
static bool isSRL16(const SDValue &Op)
static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC)
static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr, SDValue Inc, const SelectionDAG &DAG)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static Register genTPEntry(MachineBasicBlock *TpEntry, MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpExit, Register OpSizeReg, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI)
Adds logic in loop entry MBB to calculate loop iteration count and adds t2WhileLoopSetup and t2WhileL...
static SDValue createGPRPairNodei64(SelectionDAG &DAG, SDValue V)
static bool isLTorLE(ISD::CondCode CC)
static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl, SelectionDAG &DAG)
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static SDValue PerformBITCASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG)
static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG)
static bool hasNormalLoadOperand(SDNode *N)
hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node are normal,...
static SDValue PerformInsertEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
PerformInsertEltCombine - Target-specific dag combine xforms for ISD::INSERT_VECTOR_ELT.
static SDValue PerformVDUPLANECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVDUPLANECombine - Target-specific dag combine xforms for ARMISD::VDUPLANE.
static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static cl::opt< unsigned > ConstpoolPromotionMaxTotal("arm-promote-constant-max-total", cl::Hidden, cl::desc("Maximum size of ALL constants to promote into a constant pool"), cl::init(128))
static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static RTLIB::Libcall getDivRemLibcall(const SDNode *N, MVT::SimpleValueType SVT)
static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG &DAG)
SkipLoadExtensionForVMULL - return a load of the original vector size that does not do any sign/zero ...
static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static const MCPhysReg GPRArgRegs[]
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombineWithOperands - Try DAG combinations for an ADD with operands N0 and N1.
static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT, SelectionDAG &DAG)
static bool isVZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformORCombineToSMULWBT(SDNode *OR, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isVTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of "vector_shuffle v,...
static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue FindBFIToCombineWith(SDNode *N)
static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT, SelectionDAG &DAG)
static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, bool &swpCmpOps, bool &swpVselOps)
static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isS16(const SDValue &Op, SelectionDAG &DAG)
static bool isSRA16(const SDValue &Op)
constexpr MVT FlagsVT
Value type used for "flags" operands / results (either CPSR or FPSCR_NZCV).
static SDValue AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue LowerInterruptReturn(SmallVectorImpl< SDValue > &RetOps, const SDLoc &DL, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl, SelectionDAG &DAG)
static void expandf64Toi32(SDValue Op, SelectionDAG &DAG, SDValue &RetVal1, SDValue &RetVal2)
static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isSHL16(const SDValue &Op)
static bool isVEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseVEXT, unsigned &Imm)
static SDValue PerformMVEVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isTruncMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue PerformADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2)
Return the load opcode for a given load size.
static bool isLegalT2AddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
static bool isLegalMVEShuffleOp(unsigned PFEntry)
static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N, SelectionDAG &DAG)
static bool isVUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG)
PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for ISD::VECTOR_SHUFFLE.
static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG)
SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND, ANY_EXTEND,...
static bool isVMOVNTruncMask(ArrayRef< int > M, EVT ToVT, bool rev)
static SDValue PerformVQMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static MachineBasicBlock * OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ)
static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformAddcSubcCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static TargetLowering::ArgListTy getDivRemArgList(const SDNode *N, LLVMContext *Context, const ARMSubtarget *Subtarget)
static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl)
getZeroVector - Returns a vector of specified type with all zero elements.
static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static ARMCC::CondCodes getVCMPCondCode(SDValue N)
static cl::opt< bool > ARMInterworking("arm-interworking", cl::Hidden, cl::desc("Enable / disable ARM interworking (for debugging only)"), cl::init(true))
static void ReplaceREADCYCLECOUNTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformORCombineToBFI(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, SDValue &CC, bool &Invert, SDValue &OtherOp, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVSetCCToVCTPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isZeroVector(SDValue N)
static SDValue PerformAddeSubeCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void ReplaceCMP_SWAP_64Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isLowerSaturate(const SDValue LHS, const SDValue RHS, const SDValue TrueVal, const SDValue FalseVal, const ISD::CondCode CC, const SDValue K)
static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG)
static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned StSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment store operation with given size.
static bool isVMOVNMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue CombineBaseUpdate(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineBaseUpdate - Target-specific DAG combine function for VLDDUP, NEON load/store intrinsics,...
static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVRRDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMOVRRDCombine - Target-specific dag combine xforms for ARMISD::VMOVRRD.
static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain)
static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue PerformVMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMULCombine Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the special multi...
static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG)
static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformORCombine - Target-specific dag combine xforms for ISD::OR.
static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG)
static SDValue PerformTruncatingStoreCombine(StoreSDNode *St, SelectionDAG &DAG)
static unsigned SelectPairHalf(unsigned Elements, ArrayRef< int > Mask, unsigned Index)
static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned LdSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment load operation with given size.
static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG)
static bool isValidBaseUpdate(SDNode *N, SDNode *User)
static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG, const ARMSubtarget *ST, const SDLoc &dl)
static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op)
static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformXORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment, bool isSEXTLoad, bool IsMasked, bool isLE, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
std::pair< unsigned, const TargetRegisterClass * > RCPair
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,...
static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
cl::opt< unsigned > MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden, cl::desc("Maximum interleave factor for MVE VLDn to generate."), cl::init(2))
static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, unsigned SplatBitSize, SelectionDAG &DAG, const SDLoc &dl, EVT &VT, EVT VectorVT, VMOVModImmType type)
isVMOVModifiedImm - Check if the specified splat value corresponds to a valid vector constant for a N...
static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC, SelectionDAG &DAG)
BC is a bitcast that is about to be turned into a VMOVDRR.
static SDValue promoteToConstantPool(const ARMTargetLowering *TLI, const GlobalValue *GV, SelectionDAG &DAG, EVT PtrVT, const SDLoc &dl)
static unsigned isNEONTwoResultShuffleMask(ArrayRef< int > ShuffleMask, EVT VT, unsigned &WhichResult, bool &isV_UNDEF)
Check if ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN), and return the corresponding AR...
static bool BitsProperlyConcatenate(const APInt &A, const APInt &B)
static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG)
static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target, struct BaseUpdateUser &User, bool SimpleConstIncOnly, TargetLowering::DAGCombinerInfo &DCI)
static bool allUsersAreInFunction(const Value *V, const Function *F)
Return true if all users of V are within function F, looking through ConstantExprs.
static bool isSingletonVEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG)
PerformVMOVDRRCombine - Target-specific dag combine xforms for ARMISD::VMOVDRR.
static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, SDValue &SatK)
static bool isLegalAddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
isLegalAddressImmediate - Return true if the integer value can be used as the offset of the target ad...
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isLegalT1AddressImmediate(int64_t V, EVT VT)
static SDValue CombineANDShift(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSHLSimplify(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformADDECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDECombine - Target-specific dag combine transform from ARMISD::ADDC, ARMISD::ADDE,...
static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformHWLoopCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool isVUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of "vector_shuffle v,...
static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base, uint64_t &Members)
static SDValue PerformMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
static SDValue PerformANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG)
static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm, bool &Negate)
static bool canChangeToInt(SDValue Op, bool &SeenZero, const ARMSubtarget *Subtarget)
canChangeToInt - Given the fp compare operand, return true if it is suitable to morph to an integer c...
static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2)
Return the store opcode for a given store size.
static bool IsVUZPShuffleNode(SDNode *N)
static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget, MachineInstr &MI, const SDNode *Node)
Attaches vregs to MEMCPY that it will use as scratch registers when it is expanded into LDM/STM.
static bool isFloatingPointZero(SDValue Op)
isFloatingPointZero - Return true if this is +0.0.
static SDValue findMUL_LOHI(SDValue V)
static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N, SelectionDAG &DAG)
static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG)
static void genTPLoopBody(MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI, Register OpSrcReg, Register OpDestReg, Register ElementCountReg, Register TotalIterationsReg, bool IsMemcpy)
Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and t2DoLoopEnd.
static SDValue PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformBUILD_VECTORCombine - Target-specific dag combine xforms for ISD::BUILD_VECTOR.
static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformMinMaxCombine - Target-specific DAG combining for creating truncating saturates.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static Function * getFunction(Constant *C)
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
Module.h This file contains the declarations for the Module class.
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
static Value * LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctpop of V before the specified instruction IP.
loop Loop Strength Reduction
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
unsigned const TargetRegisterInfo * TRI
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
PowerPC Reduce CR logical Operation
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
This file describes how to lower LLVM code to machine code.
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Assuming the flags are set by MI(a,b), return the condition code if we modify the instructions such t...
static constexpr int Concat[]
static bool isIntrinsic(const CallBase &Call, Intrinsic::ID ID)
bool getExactInverse(APFloat *inv) const
APInt bitcastToAPInt() const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
virtual const ARMBaseRegisterInfo & getRegisterInfo() const =0
const uint32_t * getSjLjDispatchPreservedMask(const MachineFunction &MF) const
const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const override
Code Generation virtual methods...
Register getFrameRegister(const MachineFunction &MF) const override
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
const uint32_t * getTLSCallPreservedMask(const MachineFunction &MF) const
const uint32_t * getThisReturnPreservedMask(const MachineFunction &MF, CallingConv::ID) const
getThisReturnPreservedMask - Returns a call preserved mask specific to the case that 'returned' is on...
static ARMConstantPoolConstant * Create(const Constant *C, unsigned ID)
static ARMConstantPoolMBB * Create(LLVMContext &C, const MachineBasicBlock *mbb, unsigned ID, unsigned char PCAdj)
static ARMConstantPoolSymbol * Create(LLVMContext &C, StringRef s, unsigned ID, unsigned char PCAdj)
ARMConstantPoolValue - ARM specific constantpool value.
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
int getVarArgsFrameIndex() const
int getPromotedConstpoolIncrease() const
SmallPtrSet< const GlobalVariable *, 2 > & getGlobalsPromotedToConstantPool()
void setArgumentStackToRestore(unsigned v)
bool branchTargetEnforcement() const
unsigned createPICLabelUId()
void setPromotedConstpoolIncrease(int Sz)
bool isThumb1OnlyFunction() const
void setArgRegsSaveSize(unsigned s)
bool shouldSignReturnAddress() const
bool isCmseNSEntryFunction() const
void setReturnRegsCount(unsigned s)
void setVarArgsFrameIndex(int Index)
unsigned getArgRegsSaveSize() const
void markGlobalAsPromotedToConstantPool(const GlobalVariable *GV)
Indicate to the backend that GV has had its storage changed to inside a constant pool.
void setIsSplitCSR(bool s)
void setArgumentStackSize(unsigned size)
unsigned getArgumentStackSize() const
bool isTargetMachO() const
bool isTargetAEABI() const
bool supportsTailCall() const
const Triple & getTargetTriple() const
const ARMBaseInstrInfo * getInstrInfo() const override
bool isThumb1Only() const
bool hasFPARMv8Base() const
bool isTargetWindows() const
bool isGVIndirectSymbol(const GlobalValue *GV) const
True if the GV will be accessed via an indirect symbol.
const ARMTargetLowering * getTargetLowering() const override
bool isTargetDarwin() const
const ARMBaseRegisterInfo * getRegisterInfo() const override
bool isTargetAndroid() const
bool isTargetCOFF() const
bool isTargetGNUAEABI() const
bool isTargetWatchOS() const
unsigned getPreferBranchLogAlignment() const
bool useNEONForSinglePrecisionFP() const
const InstrItineraryData * getInstrItineraryData() const override
getInstrItins - Return the instruction itineraries based on subtarget selection.
bool isTargetWatchABI() const
bool hasAnyDataBarrier() const
bool isTargetDriverKit() const
bool allowsUnalignedMem() const
bool isTargetMuslAEABI() const
bool isTargetLinux() const
bool isTargetHardFloat() const
Align getDualLoadStoreAlignment() const
bool isReadOnly(const GlobalValue *GV) const
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
bool shouldInsertFencesForAtomic(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const override
Return the correct alignment for the current calling convention.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
isLegalAddressingMode - Return true if the addressing mode represented by AM is legal for this target...
const ARMSubtarget * getSubtarget() const
bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const
bool isLegalT1ScaledAddressingMode(const AddrMode &AM, EVT VT) const
Returns true if the addressing mode representing by AM is legal for the Thumb1 target,...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPreIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mod...
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
ReplaceNodeResults - Replace the results of node with an illegal result type with new values built ou...
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
bool isLegalAddImmediate(int64_t Imm) const override
isLegalAddImmediate - Return true if the specified immediate is legal add immediate,...
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
SDValue PerformMVETruncCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const override
isFPImmLegal - Returns true if the target can instruction select the specified FP immediate natively.
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
SDValue PerformIntrinsicCombine(SDNode *N, DAGCombinerInfo &DCI) const
PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
bool isDesirableToCommuteXorWithShift(const SDNode *N) const override
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
bool ExpandInlineAsm(CallInst *CI) const override
This hook allows the target to expand an inline asm call to be explicit llvm code if it wants to.
SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const
PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const override
Create the IR node for the given complex deinterleaving operation.
bool isComplexDeinterleavingSupported() const override
Does this target support complex deinterleaving.
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
SDValue PerformMVEExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
getSetCCResultType - Return the value type to use for ISD::SETCC.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool isVarArg) const
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
createFastISel - This method returns a target specific FastISel object, or null if the target does no...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
isShuffleMaskLegal - Targets can use this to indicate that they only support some VECTOR_SHUFFLE oper...
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const override
getRegClassFor - Return the register class that should be used for the specified value type.
bool useLoadStackGuardNode(const Module &M) const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vstN intrinsic.
ARMTargetLowering(const TargetMachine &TM, const ARMSubtarget &STI)
bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const override
Does this target support complex deinterleaving with the given operation and type.
SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const
PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const override
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
Instruction * makeDMB(IRBuilderBase &Builder, ARM_MB::MemBOpt Domain) const
bool isLegalICmpImmediate(int64_t Imm) const override
isLegalICmpImmediate - Return true if the specified immediate is legal icmp immediate,...
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool isVarArg) const
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
allowsMisalignedMemoryAccesses - Returns true if the target allows unaligned memory accesses of the s...
bool isLegalInterleavedAccessType(unsigned Factor, FixedVectorType *VecTy, Align Alignment, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
bool isVectorLoadExtDesirable(SDValue ExtVal) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const override
Return true if the target can combine store(extractelement VectorTy, Idx).
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vldN intrinsic.
bool useSoftFloat() const override
bool alignLoopsWithOptSize() const override
Should loops be aligned even when the function is marked OptSize (but not MinSize).
SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
LowerAsmOperandForConstraint - Lower the specified operand into the Ops vector.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const override
Returns true if an argument of type Ty needs to be passed in a contiguous block of registers in calli...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPostIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mo...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
bool isFloatingPointOperation() const
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
The address of a basic block.
static BranchProbability getZero()
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
CCState - This class holds information needed while lowering arguments and return values.
void getInRegsParamInfo(unsigned InRegsParamRecordIndex, unsigned &BeginReg, unsigned &EndReg) const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
void rewindByValRegsInfo()
unsigned getInRegsParamsProcessed() const
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void addInRegsParamInfo(unsigned RegBegin, unsigned RegEnd)
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
unsigned getInRegsParamsCount() const
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
unsigned getValNo() const
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
Value * getCalledOperand() const
AttributeList getAttributes() const
Return the attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
const APFloat & getValueAPF() const
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
MaybeAlign getStackAlignment() const
Returns the natural stack alignment, or MaybeAlign() if one wasn't specified.
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPreferredAlign(const GlobalVariable *GV) const
Returns the preferred alignment of the specified global.
StringRef getPrivateGlobalPrefix() const
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Diagnostic information for unsupported feature in backend.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
const GlobalValue * getGlobal() const
bool hasExternalWeakLinkage() const
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
bool isStrongDefinitionForLinker() const
Returns true if this global's definition will be the one chosen by the linker.
@ InternalLinkage
Rename collisions when linking (static functions).
Common base class shared among various IRBuilders.
Value * CreateZExtOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
BasicBlock * GetInsertBlock() const
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateTruncOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::optional< unsigned > getOperandCycle(unsigned ItinClassIndx, unsigned OperandIdx) const
Return the cycle for the given class and operand.
bool isEmpty() const
Returns true if there are no itineraries.
bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
static bool LowerToByteSwap(CallInst *CI)
Try to replace a call instruction with a call to a bswap intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Describe properties that are true of each instruction in the target description file.
unsigned getSchedClass() const
Return the scheduling class for this instruction.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
Wrapper class representing physical registers. Should be passed by value.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
bool isEHPad() const
Returns true if the block is a landing pad.
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool canFallThrough()
Return true if the block can implicitly transfer control to the block after it by falling off the end...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
pred_iterator pred_begin()
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void moveAfter(MachineBasicBlock *NewBefore)
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void computeMaxCallFrameSize(MachineFunction &MF, std::vector< MachineBasicBlock::iterator > *FrameSDOps=nullptr)
Computes the maximum size of a callframe.
void setAdjustsStack(bool V)
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool hasVAStart() const
Returns true if the function calls the llvm.va_start intrinsic.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
int getFunctionContextIndex() const
Return the index for the function context object.
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & reset(Property P)
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
const MachineOperand & getOperand(unsigned i) const
unsigned createJumpTableIndex(const std::vector< MachineBasicBlock * > &DestBBs)
createJumpTableIndex - Create a new jump table.
@ EK_Inline
EK_Inline - Jump table entries are emitted inline at their point of use.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
void setIsDef(bool Val=true)
Change a def to a use, or a use to a def.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class is used to represent an MLOAD node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
virtual void print(raw_ostream &OS, const Module *M) const
print - Print out the internal state of the pass.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
iterator_range< user_iterator > users()
void setFlags(SDNodeFlags NewFlags)
user_iterator user_begin() const
Provide iteration support to walk over all users of an SDNode.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getRegister(Register Reg, EVT VT)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
void addCallSiteInfo(const SDNode *Node, CallSiteInfo &&CallInfo)
Set CallSiteInfo to be associated with Node.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
void reserve(size_type NumEntries)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
StringRef - Represent a constant reference to a string, i.e.
const unsigned char * bytes_end() const
constexpr size_t size() const
size - Get the string size.
const unsigned char * bytes_begin() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
void setCmpLibcallCC(RTLIB::Libcall Call, ISD::CondCode CC)
Override the default CondCode to be used to test the result of the comparison libcall against zero.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
void setMinStackArgumentAlignment(Align Alignment)
Set the minimum stack alignment of an argument.
const TargetMachine & getTargetMachine() const
void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
virtual unsigned getMaxSupportedInterleaveFactor() const
Get the maximum supported factor for interleaved memory accesses.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
bool isPositionIndependent() const
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
bool isOSVersionLT(unsigned Major, unsigned Minor=0, unsigned Micro=0) const
Helper function for doing comparisons against version numbers included in the target triple.
bool isWindowsMSVCEnvironment() const
Checks if the environment could be MSVC.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isArrayTy() const
True if this is an instance of ArrayType.
bool isPointerTy() const
True if this is an instance of PointerType.
Type * getArrayElementType() const
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
unsigned getOperandNo() const
Return the operand # of this use in its User.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static CondCodes getOppositeCondition(CondCodes CC)
@ SECREL
Thread Pointer Offset.
@ SBREL
Section Relative (Windows TLS)
@ GOTTPOFF
Global Offset Table, PC Relative.
@ TPOFF
Global Offset Table, Thread Pointer Offset.
TOF
Target Operand Flag enum.
@ MO_NONLAZY
MO_NONLAZY - This is an independent flag, on a symbol operand "FOO" it represents a symbol which,...
@ MO_SBREL
MO_SBREL - On a symbol operand, this represents a static base relative relocation.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_GOT
MO_GOT - On a symbol operand, this represents a GOT relative relocation.
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
unsigned getAM2Offset(unsigned AM2Opc)
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned createVMOVModImm(unsigned OpCmode, unsigned Val)
int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
int getFP32FP16Imm(const APInt &Imm)
If this is a FP16Imm encoded as a fp32 value, return the 8-bit encoding for it.
AddrOpc getAM2Op(unsigned AM2Opc)
bool isBitFieldInvertedMask(unsigned v)
const unsigned FPStatusBits
const unsigned FPReservedBits
const unsigned RoundingBitsPos
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Swift
Calling convention for Swift.
@ ARM_APCS
ARM Procedure Calling Standard (obsolete, but still used on some targets).
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ ARM_AAPCS
ARM Architecture Procedure Calling Standard calling convention (aka EABI).
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
ID ArrayRef< Type * > Tys
Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool CC_ARM_APCS_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool RetCC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns true if Val1 has a lower Constant Materialization Cost than Val2.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool FastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool RetCC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
void shuffle(Iterator first, Iterator last, RNG &&g)
static std::array< MachineOperand, 2 > predOps(ARMCC::CondCodes Pred, unsigned PredReg=0)
Get the operands corresponding to the given Pred value.
bool CC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
bool CC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_Win32_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
ComplexDeinterleavingOperation
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
ComplexDeinterleavingRotation
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
static MachineOperand t1CondCodeOp(bool isDead=false)
Get the operand corresponding to the conditional code result for Thumb1.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
static MachineOperand condCodeOp(unsigned CCReg=0)
Get the operand corresponding to the conditional code result.
bool isVREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isVREVMask - Check if a vector shuffle corresponds to a VREV instruction with the specified blocksize...
unsigned gettBLXrOpcode(const MachineFunction &MF)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
unsigned convertAddSubFlagsOpcode(unsigned OldOpc)
Map pseudo instructions that imply an 'S' bit onto real opcodes.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
static const unsigned PerfectShuffleTable[6561+1]
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Load/store instruction that can be merged with a base address update.
SDNode * N
Instruction that updates a pointer.
unsigned ConstInc
Pointer increment value if it is a constant, or 0 otherwise.
SDValue Inc
Pointer increment operand.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
static constexpr roundingMode rmTowardZero
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
bool isUnknown() const
Returns true if we don't know any bits.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false)
Compute knownbits resulting from addition of LHS and RHS.
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
SmallVector< ArgRegPair, 1 > ArgRegPairs
Vector of call argument and its forwarding register.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setInRegister(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)