84#include "llvm/IR/IntrinsicsARM.h"
122#define DEBUG_TYPE "arm-isel"
125STATISTIC(NumMovwMovt,
"Number of GAs materialized with movw + movt");
126STATISTIC(NumLoopByVals,
"Number of loops generated for byval arguments");
128 "Number of constants with their storage promoted into constant pools");
132 cl::desc(
"Enable / disable ARM interworking (for debugging only)"),
137 cl::desc(
"Enable / disable promotion of unnamed_addr constants into "
142 cl::desc(
"Maximum size of constant to promote into a constant pool"),
146 cl::desc(
"Maximum size of ALL constants to promote into a constant pool"),
151 cl::desc(
"Maximum interleave factor for MVE VLDn to generate."),
156 ARM::R0, ARM::R1, ARM::R2, ARM::R3
159void ARMTargetLowering::addTypeForNEON(
MVT VT,
MVT PromotedLdStVT) {
160 if (VT != PromotedLdStVT) {
169 if (ElemTy != MVT::f64)
173 if (ElemTy == MVT::i32) {
209 VT != MVT::v2i64 && VT != MVT::v1i64)
217void ARMTargetLowering::addDRTypeForNEON(
MVT VT) {
219 addTypeForNEON(VT, MVT::f64);
222void ARMTargetLowering::addQRTypeForNEON(
MVT VT) {
224 addTypeForNEON(VT, MVT::v2f64);
227void ARMTargetLowering::setAllExpand(
MVT VT) {
240void ARMTargetLowering::addAllExtLoads(
const MVT From,
const MVT To,
241 LegalizeAction Action) {
247void ARMTargetLowering::addMVEVectorTypes(
bool HasMVEFP) {
248 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
250 for (
auto VT : IntTypes) {
324 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
325 for (
auto VT : FloatTypes) {
393 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
394 for (
auto VT : LongTypes) {
411 addAllExtLoads(MVT::v8i16, MVT::v8i8,
Legal);
412 addAllExtLoads(MVT::v4i32, MVT::v4i16,
Legal);
413 addAllExtLoads(MVT::v4i32, MVT::v4i8,
Legal);
430 for (
auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
439 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
440 for (
auto VT : pTypes) {
496 for (
int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID)
504 if (Subtarget->isThumb() && Subtarget->
hasVFP2Base() &&
505 Subtarget->
hasARMOps() && !Subtarget->useSoftFloat()) {
506 static const struct {
508 const char *
const Name;
530 { RTLIB::UO_F32,
"__unordsf2vfp",
ISD::SETNE },
539 { RTLIB::UO_F64,
"__unorddf2vfp",
ISD::SETNE },
564 for (
const auto &LC : LibraryCalls) {
584 static const struct {
586 const char *
const Name;
671 for (
const auto &LC : LibraryCalls) {
681 static const struct {
683 const char *
const Name;
686 } MemOpsLibraryCalls[] = {
694 for (
const auto &LC : MemOpsLibraryCalls) {
704 static const struct {
706 const char *
const Name;
719 for (
const auto &LC : LibraryCalls) {
751 static const struct {
753 const char *
const Name;
761 for (
const auto &LC : LibraryCalls) {
772 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only() &&
773 Subtarget->hasFPRegs()) {
783 setAllExpand(MVT::f32);
784 if (!Subtarget->hasFP64())
785 setAllExpand(MVT::f64);
788 if (Subtarget->hasFullFP16()) {
797 if (Subtarget->hasBF16()) {
799 setAllExpand(MVT::bf16);
800 if (!Subtarget->hasFullFP16())
807 addAllExtLoads(VT, InnerVT,
Expand);
822 if (Subtarget->hasMVEIntegerOps())
823 addMVEVectorTypes(Subtarget->hasMVEFloatOps());
826 if (Subtarget->hasLOB()) {
830 if (Subtarget->hasNEON()) {
831 addDRTypeForNEON(MVT::v2f32);
832 addDRTypeForNEON(MVT::v8i8);
833 addDRTypeForNEON(MVT::v4i16);
834 addDRTypeForNEON(MVT::v2i32);
835 addDRTypeForNEON(MVT::v1i64);
837 addQRTypeForNEON(MVT::v4f32);
838 addQRTypeForNEON(MVT::v2f64);
839 addQRTypeForNEON(MVT::v16i8);
840 addQRTypeForNEON(MVT::v8i16);
841 addQRTypeForNEON(MVT::v4i32);
842 addQRTypeForNEON(MVT::v2i64);
844 if (Subtarget->hasFullFP16()) {
845 addQRTypeForNEON(MVT::v8f16);
846 addDRTypeForNEON(MVT::v4f16);
849 if (Subtarget->hasBF16()) {
850 addQRTypeForNEON(MVT::v8bf16);
851 addDRTypeForNEON(MVT::v4bf16);
855 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
894 if (Subtarget->hasNEON()) {
1006 for (
MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
1015 for (
auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
1024 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
1032 if (Subtarget->hasMVEIntegerOps()) {
1037 if (Subtarget->hasMVEFloatOps()) {
1041 if (!Subtarget->hasFP64()) {
1088 if (Subtarget->hasFullFP16()) {
1094 if (!Subtarget->hasFP16()) {
1142 if (Subtarget->hasDSP()) {
1164 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6Ops()
1165 || (Subtarget->
isThumb2() && !Subtarget->hasDSP()))
1180 if (Subtarget->hasMVEIntegerOps())
1190 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops())
1201 if (!Subtarget->hasV5TOps() || Subtarget->
isThumb1Only()) {
1210 if (Subtarget->hasPerfMon())
1214 if (!Subtarget->hasV6Ops())
1217 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1218 : Subtarget->hasDivideInARMMode();
1225 if (Subtarget->
isTargetWindows() && !Subtarget->hasDivideInThumbMode()) {
1242 HasStandaloneRem =
false;
1247 const char *
const Name;
1249 } LibraryCalls[] = {
1261 for (
const auto &LC : LibraryCalls) {
1268 const char *
const Name;
1270 } LibraryCalls[] = {
1282 for (
const auto &LC : LibraryCalls) {
1326 InsertFencesForAtomic =
false;
1328 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
1332 if (!Subtarget->isThumb() || !Subtarget->
isMClass())
1337 if (!Subtarget->hasAcquireRelease() ||
1340 InsertFencesForAtomic =
true;
1346 if (Subtarget->hasDataBarrier())
1347 InsertFencesForAtomic =
true;
1367 if (!InsertFencesForAtomic) {
1375 (!Subtarget->
isMClass() && Subtarget->hasV6Ops())) {
1387 }
else if ((Subtarget->
isMClass() && Subtarget->hasV8MBaselineOps()) ||
1388 Subtarget->hasForced32BitAtomics()) {
1402 if (!Subtarget->hasV6Ops()) {
1408 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1440 if (Subtarget->hasFullFP16()) {
1450 if (Subtarget->hasFullFP16())
1465 if (!Subtarget->useSoftFloat() && Subtarget->
hasVFP2Base() &&
1479 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only()) {
1487 if (!Subtarget->hasFP16()) {
1518 if (Subtarget->hasNEON()) {
1525 if (Subtarget->hasFP64()) {
1538 if (Subtarget->hasFullFP16()) {
1556 if (Subtarget->hasNEON()) {
1568 if (Subtarget->hasFullFP16()) {
1586 if (Subtarget->hasMVEIntegerOps())
1589 if (Subtarget->hasV6Ops())
1594 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1601 if (Subtarget->useSoftFloat() || Subtarget->
isThumb1Only() ||
1627 if (Subtarget->isThumb() || Subtarget->
isThumb2())
1632 return Subtarget->useSoftFloat();
1645std::pair<const TargetRegisterClass *, uint8_t>
1656 case MVT::f32:
case MVT::f64:
case MVT::v8i8:
case MVT::v4i16:
1657 case MVT::v2i32:
case MVT::v1i64:
case MVT::v2f32:
1658 RRC = &ARM::DPRRegClass;
1666 case MVT::v16i8:
case MVT::v8i16:
case MVT::v4i32:
case MVT::v2i64:
1667 case MVT::v4f32:
case MVT::v2f64:
1668 RRC = &ARM::DPRRegClass;
1672 RRC = &ARM::DPRRegClass;
1676 RRC = &ARM::DPRRegClass;
1680 return std::make_pair(RRC,
Cost);
1684#define MAKE_CASE(V) \
1904 if ((Subtarget->hasMVEIntegerOps() &&
1905 (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
1906 VT == MVT::v16i8)) ||
1907 (Subtarget->hasMVEFloatOps() &&
1908 (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16)))
1922 if (Subtarget->hasNEON()) {
1923 if (VT == MVT::v4i64)
1924 return &ARM::QQPRRegClass;
1925 if (VT == MVT::v8i64)
1926 return &ARM::QQQQPRRegClass;
1928 if (Subtarget->hasMVEIntegerOps()) {
1929 if (VT == MVT::v4i64)
1930 return &ARM::MQQPRRegClass;
1931 if (VT == MVT::v8i64)
1932 return &ARM::MQQQQPRRegClass;
1941 Align &PrefAlign)
const {
1942 if (!isa<MemIntrinsic>(CI))
1960 unsigned NumVals =
N->getNumValues();
1964 for (
unsigned i = 0; i != NumVals; ++i) {
1965 EVT VT =
N->getValueType(i);
1966 if (VT == MVT::Glue || VT == MVT::Other)
1972 if (!
N->isMachineOpcode())
1996 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
1997 return Const->getZExtValue() == 16;
2004 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2005 return Const->getZExtValue() == 16;
2012 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2013 return Const->getZExtValue() == 16;
2082 bool isVarArg)
const {
2103 else if (Subtarget->hasFPRegs() && !Subtarget->
isThumb1Only() &&
2124 bool isVarArg)
const {
2125 return CCAssignFnForNode(
CC,
false, isVarArg);
2129 bool isVarArg)
const {
2130 return CCAssignFnForNode(
CC,
true, isVarArg);
2137 bool isVarArg)
const {
2138 switch (getEffectiveCallingConv(
CC, isVarArg)) {
2164 if (Subtarget->hasFullFP16()) {
2177 if (Subtarget->hasFullFP16()) {
2191SDValue ARMTargetLowering::LowerCallResult(
2203 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
2208 if (i == 0 && isThisReturn) {
2210 "unexpected return calling convention register assignment");
2221 Chain =
Lo.getValue(1);
2222 InGlue =
Lo.getValue(2);
2226 Chain =
Hi.getValue(1);
2227 InGlue =
Hi.getValue(2);
2239 Chain =
Lo.getValue(1);
2240 InGlue =
Lo.getValue(2);
2243 Chain =
Hi.getValue(1);
2244 InGlue =
Hi.getValue(2);
2279std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
2281 bool IsTailCall,
int SPDiff)
const {
2303 return std::make_pair(DstAddr, DstInfo);
2308 RegsToPassVector &RegsToPass,
2315 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
2316 unsigned id = Subtarget->
isLittle() ? 0 : 1;
2329 std::tie(DstAddr, DstInfo) =
2330 computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
2362 bool isStructRet = (Outs.
empty()) ?
false : Outs[0].
Flags.isSRet();
2363 bool isThisReturn =
false;
2364 bool isCmseNSCall =
false;
2365 bool isSibCall =
false;
2366 bool PreferIndirect =
false;
2367 bool GuardWithBTI =
false;
2371 !Subtarget->noBTIAtReturnTwice())
2376 isCmseNSCall =
true;
2388 if (isa<GlobalAddressSDNode>(Callee)) {
2392 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
2395 PreferIndirect = Subtarget->isThumb() && Subtarget->
hasMinSize() &&
2397 return isa<Instruction>(U) &&
2398 cast<Instruction>(U)->getParent() == BB;
2404 isTailCall = IsEligibleForTailCallOptimization(
2405 Callee, CallConv, isVarArg, isStructRet,
2421 "site marked musttail");
2429 unsigned NumBytes = CCInfo.getStackSize();
2438 if (isTailCall && !isSibCall) {
2445 NumBytes =
alignTo(NumBytes, StackAlign);
2450 SPDiff = NumReusableBytes - NumBytes;
2454 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (
unsigned)-SPDiff)
2470 RegsToPassVector RegsToPass;
2478 bool AfterFormalArgLoads =
false;
2482 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.
size();
2484 ++i, ++realArgIdx) {
2486 SDValue Arg = OutVals[realArgIdx];
2488 bool isByVal =
Flags.isByVal();
2508 if (isTailCall && VA.
isMemLoc() && !AfterFormalArgLoads) {
2510 AfterFormalArgLoads =
true;
2522 auto ArgVT = Outs[realArgIdx].ArgVT;
2523 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2541 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2542 StackPtr, MemOpChains, isTailCall, SPDiff);
2546 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2547 StackPtr, MemOpChains, isTailCall, SPDiff);
2552 std::tie(DstAddr, DstInfo) =
2553 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2557 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2558 StackPtr, MemOpChains, isTailCall, SPDiff);
2560 if (realArgIdx == 0 &&
Flags.isReturned() && !
Flags.isSwiftSelf() &&
2561 Outs[0].VT == MVT::i32) {
2563 "unexpected calling convention register assignment");
2564 assert(!
Ins.empty() && Ins[0].VT == MVT::i32 &&
2565 "unexpected use of 'returned'");
2566 isThisReturn =
true;
2570 CSInfo.ArgRegPairs.emplace_back(VA.
getLocReg(), i);
2571 RegsToPass.push_back(std::make_pair(VA.
getLocReg(), Arg));
2572 }
else if (isByVal) {
2574 unsigned offset = 0;
2578 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2579 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2581 if (CurByValIdx < ByValArgsCount) {
2583 unsigned RegBegin, RegEnd;
2584 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
2589 for (i = 0, j = RegBegin;
j < RegEnd; i++,
j++) {
2596 RegsToPass.push_back(std::make_pair(j, Load));
2601 offset = RegEnd - RegBegin;
2603 CCInfo.nextInRegsParam();
2606 if (
Flags.getByValSize() > 4*offset) {
2610 std::tie(Dst, DstInfo) =
2611 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2620 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2628 std::tie(DstAddr, DstInfo) =
2629 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2636 if (!MemOpChains.
empty())
2642 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
2643 Chain = DAG.
getCopyToReg(Chain, dl, RegsToPass[i].first,
2644 RegsToPass[i].second, InGlue);
2651 bool isDirect =
false;
2656 GVal =
G->getGlobal();
2657 bool isStub = !
TM.shouldAssumeDSOLocal(GVal) && Subtarget->
isTargetMachO();
2659 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->
isMClass());
2660 bool isLocalARMFunc =
false;
2663 if (Subtarget->genLongCalls()) {
2665 "long-calls codegen is not position independent!");
2669 if (isa<GlobalAddressSDNode>(Callee)) {
2670 if (Subtarget->genExecuteOnly()) {
2689 const char *
Sym = S->getSymbol();
2691 if (Subtarget->genExecuteOnly()) {
2710 }
else if (isa<GlobalAddressSDNode>(Callee)) {
2711 if (!PreferIndirect) {
2716 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !
ARMInterworking);
2718 if (isStub && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2730 "Windows is the only supported COFF target");
2734 else if (!
TM.shouldAssumeDSOLocal(GVal))
2750 const char *
Sym = S->getSymbol();
2751 if (isARMFunc && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2755 ARMPCLabelIndex, 4);
2769 assert(!isARMFunc && !isDirect &&
2770 "Cannot handle call to ARM function or direct call");
2773 "call to non-secure function would "
2774 "require passing arguments on stack",
2781 "call to non-secure function would return value through pointer",
2789 if (Subtarget->isThumb()) {
2792 else if (isCmseNSCall)
2794 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2799 if (!isDirect && !Subtarget->hasV5TOps())
2801 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2814 if (isTailCall && !isSibCall) {
2819 std::vector<SDValue> Ops;
2820 Ops.push_back(Chain);
2821 Ops.push_back(Callee);
2829 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2830 Ops.push_back(DAG.
getRegister(RegsToPass[i].first,
2831 RegsToPass[i].second.getValueType()));
2843 isThisReturn =
false;
2849 assert(Mask &&
"Missing call preserved mask for calling convention");
2853 Ops.push_back(InGlue);
2865 Chain = DAG.
getNode(CallOpc, dl, NodeTys, Ops);
2877 Chain = DAG.
getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
2883 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2884 InVals, isThisReturn,
2885 isThisReturn ? OutVals[0] :
SDValue());
2892void ARMTargetLowering::HandleByVal(
CCState *State,
unsigned &
Size,
2893 Align Alignment)
const {
2895 Alignment = std::max(Alignment,
Align(4));
2901 unsigned AlignInRegs = Alignment.
value() / 4;
2902 unsigned Waste = (ARM::R4 -
Reg) % AlignInRegs;
2903 for (
unsigned i = 0; i < Waste; ++i)
2909 unsigned Excess = 4 * (ARM::R4 -
Reg);
2916 if (NSAAOffset != 0 &&
Size > Excess) {
2928 unsigned ByValRegBegin =
Reg;
2929 unsigned ByValRegEnd = std::min<unsigned>(Reg +
Size / 4, ARM::R4);
2933 for (
unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2939 Size = std::max<int>(
Size - Excess, 0);
2950 int FI = std::numeric_limits<int>::max();
2958 if (!Flags.isByVal()) {
2964 }
else if (
LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
2965 if (Flags.isByVal())
2980 assert(FI != std::numeric_limits<int>::max());
2989bool ARMTargetLowering::IsEligibleForTailCallOptimization(
2991 bool isCalleeStructRet,
bool isCallerStructRet,
2995 const bool isIndirect)
const {
3008 if (Outs.
size() >= 4 &&
3009 (!isa<GlobalAddressSDNode>(
Callee.getNode()) || isIndirect)) {
3027 return CalleeCC == CallerCC;
3031 if (isCalleeStructRet || isCallerStructRet)
3045 (!
TT.isOSWindows() ||
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO()))
3052 getEffectiveCallingConv(CalleeCC, isVarArg),
3053 getEffectiveCallingConv(CallerCC, CallerF.
isVarArg()), MF,
C, Ins,
3059 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
3060 if (CalleeCC != CallerCC) {
3061 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
3062 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3075 if (!Outs.
empty()) {
3079 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs,
C);
3081 if (CCInfo.getStackSize()) {
3087 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.
size();
3089 ++i, ++realArgIdx) {
3092 SDValue Arg = OutVals[realArgIdx];
3096 if (VA.
needsCustom() && (RegVT == MVT::f64 || RegVT == MVT::v2f64)) {
3103 if (!ArgLocs[++i].isRegLoc())
3105 if (RegVT == MVT::v2f64) {
3106 if (!ArgLocs[++i].isRegLoc())
3108 if (!ArgLocs[++i].isRegLoc())
3133 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
3142 StringRef IntKind =
F.getFnAttribute(
"interrupt").getValueAsString();
3155 if (IntKind ==
"" || IntKind ==
"IRQ" || IntKind ==
"FIQ" ||
3158 else if (IntKind ==
"SWI" || IntKind ==
"UNDEF")
3162 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
3189 bool isLittleEndian = Subtarget->
isLittle();
3201 "secure entry function would return value through pointer",
3207 for (
unsigned i = 0, realRVLocIdx = 0;
3209 ++i, ++realRVLocIdx) {
3213 SDValue Arg = OutVals[realRVLocIdx];
3214 bool ReturnF16 =
false;
3249 auto RetVT = Outs[realRVLocIdx].ArgVT;
3271 DAG.
getVTList(MVT::i32, MVT::i32), Half);
3275 HalfGPRs.
getValue(isLittleEndian ? 0 : 1), Glue);
3281 HalfGPRs.
getValue(isLittleEndian ? 1 : 0), Glue);
3293 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
3295 fmrrd.
getValue(isLittleEndian ? 0 : 1), Glue);
3300 fmrrd.
getValue(isLittleEndian ? 1 : 0), Glue);
3344 return DAG.
getNode(RetNode, dl, MVT::Other, RetOps);
3347bool ARMTargetLowering::isUsedByReturnOnly(
SDNode *
N,
SDValue &Chain)
const {
3348 if (
N->getNumValues() != 1)
3350 if (!
N->hasNUsesOfValue(1, 0))
3358 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3360 TCChain =
Copy->getOperand(0);
3374 SDValue UseChain =
U->getOperand(0);
3382 if (
U->getOperand(
U->getNumOperands() - 1).getValueType() == MVT::Glue)
3390 if (!
Copy->hasOneUse())
3397 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3399 TCChain =
Copy->getOperand(0);
3404 bool HasRet =
false;
3419bool ARMTargetLowering::mayBeEmittedAsTailCall(
const CallInst *CI)
const {
3437 &&
"LowerWRITE_REGISTER called for non-i64 type argument.");
3453 EVT PtrVT =
Op.getValueType();
3463 if (Subtarget->genExecuteOnly()) {
3465 auto T =
const_cast<Type*
>(
CP->getType());
3466 auto C =
const_cast<Constant*
>(
CP->getConstVal());
3477 return LowerGlobalAddress(GA, DAG);
3482 Align CPAlign =
CP->getAlign();
3484 CPAlign = std::max(CPAlign,
Align(4));
3485 if (
CP->isMachineConstantPoolEntry())
3497 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3506 unsigned ARMPCLabelIndex = 0;
3509 const BlockAddress *BA = cast<BlockAddressSDNode>(
Op)->getBlockAddress();
3512 if (!IsPositionIndependent) {
3515 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3526 if (!IsPositionIndependent)
3557ARMTargetLowering::LowerGlobalTLSAddressDarwin(
SDValue Op,
3560 "This function expects a Darwin target");
3565 SDValue DescAddr = LowerGlobalAddressDarwin(
Op, DAG);
3571 MVT::i32,
DL, Chain, DescAddr,
3595 Chain, FuncTLVGet, DAG.
getRegister(ARM::R0, MVT::i32),
3601ARMTargetLowering::LowerGlobalTLSAddressWindows(
SDValue Op,
3618 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
3645 const auto *GA = cast<GlobalAddressSDNode>(
Op);
3662 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3684 Args.push_back(Entry);
3692 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
3693 return CallResult.first;
3715 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3723 PtrVT, dl, Chain,
Offset,
3725 Chain =
Offset.getValue(1);
3731 PtrVT, dl, Chain,
Offset,
3741 PtrVT, dl, Chain,
Offset,
3757 return LowerGlobalTLSAddressDarwin(
Op, DAG);
3760 return LowerGlobalTLSAddressWindows(
Op, DAG);
3769 return LowerToTLSGeneralDynamicModel(GA, DAG);
3772 return LowerToTLSExecModels(GA, DAG, model);
3781 while (!Worklist.
empty()) {
3783 if (isa<ConstantExpr>(U)) {
3788 auto *
I = dyn_cast<Instruction>(U);
3789 if (!
I ||
I->getParent()->getParent() !=
F)
3817 auto *GVar = dyn_cast<GlobalVariable>(GV);
3818 if (!GVar || !GVar->hasInitializer() ||
3819 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3820 !GVar->hasLocalLinkage())
3825 auto *
Init = GVar->getInitializer();
3827 Init->needsDynamicRelocation())
3836 auto *CDAInit = dyn_cast<ConstantDataArray>(
Init);
3839 unsigned RequiredPadding = 4 - (
Size % 4);
3840 bool PaddingPossible =
3841 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3846 unsigned PaddedSize =
Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3870 if (RequiredPadding != 4) {
3875 while (RequiredPadding--)
3887 ++NumConstpoolPromoted;
3892 if (
const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
3893 if (!(GV = GA->getAliaseeObject()))
3895 if (
const auto *V = dyn_cast<GlobalVariable>(GV))
3896 return V->isConstant();
3897 return isa<Function>(GV);
3905 return LowerGlobalAddressWindows(
Op, DAG);
3907 return LowerGlobalAddressELF(
Op, DAG);
3909 return LowerGlobalAddressDarwin(
Op, DAG);
3917 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
3921 if (GV->
isDSOLocal() && !Subtarget->genExecuteOnly())
3934 }
else if (Subtarget->
isROPI() && IsRO) {
3939 }
else if (Subtarget->
isRWPI() && !IsRO) {
3964 if (Subtarget->
useMovt() || Subtarget->genExecuteOnly()) {
3983 "ROPI/RWPI not currently supported for Darwin");
3986 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4009 "Windows on ARM expects to use movw/movt");
4011 "ROPI/RWPI not currently supported for Windows");
4014 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4018 else if (!
TM.shouldAssumeDSOLocal(GV))
4042 DAG.
getVTList(MVT::i32, MVT::Other),
Op.getOperand(0),
4043 Op.getOperand(1), Val);
4060SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
4063 Op.getConstantOperandVal(
Op.getOperand(0).getValueType() == MVT::Other);
4067 case Intrinsic::arm_gnu_eabi_mcount: {
4076 assert(Mask &&
"Missing call preserved mask for calling convention");
4081 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
4085 if (Subtarget->isThumb())
4088 ARM::tBL_PUSHLR, dl, ResultTys,
4089 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
4090 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
4094 {ReturnAddress, Callee, RegisterMask, Chain}),
4103 unsigned IntNo =
Op.getConstantOperandVal(0);
4107 case Intrinsic::thread_pointer: {
4111 case Intrinsic::arm_cls: {
4112 const SDValue &Operand =
Op.getOperand(1);
4113 const EVT VTy =
Op.getValueType();
4124 case Intrinsic::arm_cls64: {
4127 const SDValue &Operand =
Op.getOperand(1);
4128 const EVT VTy =
Op.getValueType();
4151 case Intrinsic::eh_sjlj_lsda: {
4158 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
4168 if (IsPositionIndependent) {
4174 case Intrinsic::arm_neon_vabs:
4177 case Intrinsic::arm_neon_vmulls:
4178 case Intrinsic::arm_neon_vmullu: {
4179 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
4182 Op.getOperand(1),
Op.getOperand(2));
4184 case Intrinsic::arm_neon_vminnm:
4185 case Intrinsic::arm_neon_vmaxnm: {
4186 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
4189 Op.getOperand(1),
Op.getOperand(2));
4191 case Intrinsic::arm_neon_vminu:
4192 case Intrinsic::arm_neon_vmaxu: {
4193 if (
Op.getValueType().isFloatingPoint())
4195 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
4198 Op.getOperand(1),
Op.getOperand(2));
4200 case Intrinsic::arm_neon_vmins:
4201 case Intrinsic::arm_neon_vmaxs: {
4203 if (!
Op.getValueType().isFloatingPoint()) {
4204 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4207 Op.getOperand(1),
Op.getOperand(2));
4209 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4212 Op.getOperand(1),
Op.getOperand(2));
4214 case Intrinsic::arm_neon_vtbl1:
4216 Op.getOperand(1),
Op.getOperand(2));
4217 case Intrinsic::arm_neon_vtbl2:
4219 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4220 case Intrinsic::arm_mve_pred_i2v:
4221 case Intrinsic::arm_mve_pred_v2i:
4224 case Intrinsic::arm_mve_vreinterpretq:
4227 case Intrinsic::arm_mve_lsll:
4229 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4230 case Intrinsic::arm_mve_asrl:
4232 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4243 if (!Subtarget->hasDataBarrier()) {
4247 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4248 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4259 }
else if (Subtarget->preferISHSTBarriers() &&
4268 DAG.
getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4276 (!Subtarget->
isThumb1Only() && Subtarget->hasV5TEOps())))
4278 return Op.getOperand(0);
4281 unsigned isRead =
~Op.getConstantOperandVal(2) & 1;
4283 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4285 return Op.getOperand(0);
4287 unsigned isData =
Op.getConstantOperandVal(4);
4288 if (Subtarget->isThumb()) {
4290 isRead = ~isRead & 1;
4291 isData = ~isData & 1;
4308 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
4309 return DAG.
getStore(
Op.getOperand(0), dl, FR,
Op.getOperand(1),
4317 const SDLoc &dl)
const {
4323 RC = &ARM::tGPRRegClass;
4325 RC = &ARM::GPRRegClass;
4339 MVT::i32, dl, Root, FIN,
4360 const Value *OrigArg,
4361 unsigned InRegsParamRecordIdx,
4362 int ArgOffset,
unsigned ArgSize)
const {
4377 unsigned RBegin, REnd;
4387 ArgOffset = -4 * (ARM::R4 - RBegin);
4397 for (
unsigned Reg = RBegin, i = 0;
Reg < REnd; ++
Reg, ++i) {
4406 if (!MemOps.
empty())
4415 unsigned TotalArgRegsSaveSize,
4416 bool ForceMutable)
const {
4427 CCInfo.
getStackSize(), std::max(4U, TotalArgRegsSaveSize));
4431bool ARMTargetLowering::splitValueIntoRegisterParts(
4433 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
4435 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4447SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4449 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
4450 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4463SDValue ARMTargetLowering::LowerFormalArguments(
4481 unsigned CurArgIdx = 0;
4493 unsigned ArgRegBegin = ARM::R4;
4494 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4501 if (!
Flags.isByVal())
4505 unsigned RBegin, REnd;
4507 ArgRegBegin = std::min(ArgRegBegin, RBegin);
4513 int lastInsIndex = -1;
4517 ArgRegBegin = std::min(ArgRegBegin, (
unsigned)
GPRArgRegs[RegIdx]);
4520 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4524 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4526 if (Ins[VA.
getValNo()].isOrigArg()) {
4527 std::advance(CurOrigArg,
4528 Ins[VA.
getValNo()].getOrigArgIndex() - CurArgIdx);
4539 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4546 MVT::f64, dl, Chain, FIN,
4549 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4557 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4561 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4562 RC = &ARM::HPRRegClass;
4563 else if (RegVT == MVT::f32)
4564 RC = &ARM::SPRRegClass;
4565 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4566 RegVT == MVT::v4bf16)
4567 RC = &ARM::DPRRegClass;
4568 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4569 RegVT == MVT::v8bf16)
4570 RC = &ARM::QPRRegClass;
4571 else if (RegVT == MVT::i32)
4573 : &ARM::GPRRegClass;
4620 assert(VA.
getValVT() != MVT::i64 &&
"i64 should already be lowered");
4626 if (index != lastInsIndex)
4634 if (
Flags.isByVal()) {
4635 assert(Ins[index].isOrigArg() &&
4636 "Byval arguments cannot be implicit");
4640 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4655 lastInsIndex = index;
4662 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.
getStackSize(),
4663 TotalArgRegsSaveSize);
4667 "secure entry function must not be variadic", dl.
getDebugLoc());
4678 StackArgSize =
alignTo(StackArgSize,
DL.getStackAlignment());
4687 "secure entry function requires arguments on stack", dl.
getDebugLoc());
4697 return CFP->getValueAPF().isPosZero();
4701 SDValue WrapperOp =
Op.getOperand(1).getOperand(0);
4703 if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
4704 return CFP->getValueAPF().isPosZero();
4707 Op->getValueType(0) == MVT::f64) {
4722 const SDLoc &dl)
const {
4724 unsigned C = RHSC->getZExtValue();
4790 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(RHS) &&
4792 unsigned Mask =
LHS.getConstantOperandVal(1);
4793 auto *RHSC = cast<ConstantSDNode>(
RHS.getNode());
4794 uint64_t RHSV = RHSC->getZExtValue();
4795 if (
isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4797 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4812 isa<ConstantSDNode>(RHS) &&
RHS->getAsZExtVal() == 0x80000000U &&
4814 LHS.getConstantOperandVal(1) < 31) {
4815 unsigned ShiftAmt =
LHS.getConstantOperandVal(1) + 1;
4855 return DAG.
getNode(CompareType, dl, MVT::Glue, LHS, RHS);
4861 bool Signaling)
const {
4862 assert(Subtarget->hasFP64() ||
RHS.getValueType() != MVT::f64);
4866 dl, MVT::Glue, LHS, RHS);
4869 dl, MVT::Glue, LHS);
4877 unsigned Opc =
Cmp.getOpcode();
4880 return DAG.
getNode(Opc,
DL, MVT::Glue,
Cmp.getOperand(0),
Cmp.getOperand(1));
4884 Opc =
Cmp.getOpcode();
4898std::pair<SDValue, SDValue>
4901 assert(
Op.getValueType() == MVT::i32 &&
"Unsupported value type");
4913 switch (
Op.getOpcode()) {
4926 DAG.
getVTList(
Op.getValueType(), MVT::i32), LHS, RHS)
4965 return std::make_pair(
Value, OverflowCmp);
4976 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Op, DAG, ARMcc);
4982 EVT VT =
Op.getValueType();
4985 ARMcc, CCR, OverflowCmp);
5025 EVT VT =
Op.getValueType();
5029 switch (
Op.getOpcode()) {
5054 EVT VT =
Op.getValueType();
5055 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->
isThumb1Only())
5065 switch (
Op->getOpcode()) {
5081 switch (
Op->getOpcode()) {
5100 DAG.
getNode(NewOpcode, dl, MVT::i32,
5111 unsigned Opc =
Cond.getOpcode();
5113 if (
Cond.getResNo() == 1 &&
5121 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5123 EVT VT =
Op.getValueType();
5125 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
5136 dyn_cast<ConstantSDNode>(
Cond.getOperand(0));
5138 dyn_cast<ConstantSDNode>(
Cond.getOperand(1));
5140 if (CMOVTrue && CMOVFalse) {
5146 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5148 False = SelectFalse;
5149 }
else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5155 EVT VT =
Op.getValueType();
5160 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG);
5176 bool &swpCmpOps,
bool &swpVselOps) {
5204 swpCmpOps = !swpCmpOps;
5205 swpVselOps = !swpVselOps;
5228 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5230 DAG.
getVTList(MVT::i32, MVT::i32), FalseVal);
5232 DAG.
getVTList(MVT::i32, MVT::i32), TrueVal);
5242 ARMcc, CCR, duplicateCmp(Cmp, DAG));
5269 ((K ==
LHS && K == TrueVal) || (K ==
RHS && K == FalseVal))) ||
5271 ((K ==
RHS && K == TrueVal) || (K ==
LHS && K == FalseVal)));
5292 EVT VT =
Op.getValueType();
5299 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
5314 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5322 if (!isa<ConstantSDNode>(K1) || !isa<ConstantSDNode>(K2))
5325 int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue();
5326 int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue();
5327 int64_t PosVal = std::max(Val1, Val2);
5328 int64_t NegVal = std::min(Val1, Val2);
5374 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
5375 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5380 if (*K != KTmp || V != VTmp)
5391bool ARMTargetLowering::isUnsupportedFloatingType(
EVT VT)
const {
5395 return !Subtarget->hasFP64();
5397 return !Subtarget->hasFullFP16();
5402 EVT VT =
Op.getValueType();
5406 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->
isThumb2())
5418 if (VT == MVT::i32 &&
5438 if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
5439 LHS.getValueType() == MVT::i32 &&
RHS.getValueType() == MVT::i32) {
5442 unsigned Opcode = 0;
5444 if (TVal == ~FVal) {
5446 }
else if (TVal == ~FVal + 1) {
5448 }
else if (TVal + 1 == FVal) {
5450 }
else if (TVal == FVal + 1) {
5482 return DAG.
getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
5486 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5488 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5492 if (!
RHS.getNode()) {
5498 if (
LHS.getValueType() == MVT::i32) {
5510 TrueVal.getValueType() == MVT::f32 ||
5511 TrueVal.getValueType() == MVT::f64)) {
5526 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5538 (
TrueVal.getValueType() == MVT::f16 ||
5539 TrueVal.getValueType() == MVT::f32 ||
5540 TrueVal.getValueType() == MVT::f64)) {
5541 bool swpCmpOps =
false;
5542 bool swpVselOps =
false;
5557 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5561 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
5562 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
5572 if (!
N->hasOneUse())
5575 if (!
N->getNumValues())
5577 EVT VT =
Op.getValueType();
5578 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5595 return DAG.
getLoad(MVT::i32,
SDLoc(
Op), Ld->getChain(), Ld->getBasePtr(),
5596 Ld->getPointerInfo(), Ld->getAlign(),
5597 Ld->getMemOperand()->getFlags());
5615 DAG.
getLoad(MVT::i32, dl, Ld->getChain(),
Ptr, Ld->getPointerInfo(),
5616 Ld->getAlign(), Ld->getMemOperand()->getFlags());
5618 EVT PtrType =
Ptr.getValueType();
5621 RetVal2 = DAG.
getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5622 Ld->getPointerInfo().getWithOffset(4),
5624 Ld->getMemOperand()->getFlags());
5642 bool LHSSeenZero =
false;
5644 bool RHSSeenZero =
false;
5646 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5657 if (
LHS.getValueType() == MVT::f32) {
5665 Chain, Dest, ARMcc, CCR, Cmp);
5677 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5692 unsigned Opc =
Cond.getOpcode();
5695 if (
Cond.getResNo() == 1 &&
5705 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5729 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5731 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5735 if (!
RHS.getNode()) {
5743 unsigned Opc =
LHS.getOpcode();
5757 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
LHS.getValue(0), DAG, ARMcc);
5772 if (
LHS.getValueType() == MVT::i32) {
5777 Chain, Dest, ARMcc, CCR, Cmp);
5783 if (
SDValue Result = OptimizeVFPBrcond(
Op, DAG))
5794 SDValue Ops[] = { Chain, Dest, ARMcc, CCR,
Cmp };
5816 if (Subtarget->
isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5822 Addr,
Op.getOperand(2), JTI);
5828 Chain =
Addr.getValue(1);
5835 Chain =
Addr.getValue(1);
5841 EVT VT =
Op.getValueType();
5844 if (
Op.getValueType().getVectorElementType() == MVT::i32) {
5845 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
5853 const EVT OpTy =
Op.getOperand(0).getValueType();
5854 if (OpTy == MVT::v4f32)
5856 else if (OpTy == MVT::v4f16 && HasFullFP16)
5858 else if (OpTy == MVT::v8f16 && HasFullFP16)
5863 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5866 Op = DAG.
getNode(
Op.getOpcode(), dl, NewTy,
Op.getOperand(0));
5871 EVT VT =
Op.getValueType();
5875 bool IsStrict =
Op->isStrictFPOpcode();
5876 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
5878 if (isUnsupportedFloatingType(SrcVal.
getValueType())) {
5888 MakeLibCallOptions CallOptions;
5891 std::tie(Result, Chain) =
makeLibCall(DAG, LC,
Op.getValueType(), SrcVal,
5892 CallOptions, Loc, Chain);
5902 Loc,
Op.getValueType(), SrcVal);
5911 EVT VT =
Op.getValueType();
5912 EVT ToVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
5913 EVT FromVT =
Op.getOperand(0).getValueType();
5915 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5917 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5918 Subtarget->hasFP64())
5920 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5921 Subtarget->hasFullFP16())
5923 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5924 Subtarget->hasMVEFloatOps())
5926 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5927 Subtarget->hasMVEFloatOps())
5930 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5947 EVT VT =
Op.getValueType();
5950 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
5956 assert((
Op.getOperand(0).getValueType() == MVT::v4i16 ||
5957 Op.getOperand(0).getValueType() == MVT::v8i16) &&
5958 "Invalid type for custom lowering!");
5963 if (VT == MVT::v4f32)
5964 DestVecType = MVT::v4i32;
5965 else if (VT == MVT::v4f16 && HasFullFP16)
5966 DestVecType = MVT::v4i16;
5967 else if (VT == MVT::v8f16 && HasFullFP16)
5968 DestVecType = MVT::v8i16;
5974 switch (
Op.getOpcode()) {
5986 Op = DAG.
getNode(CastOpc, dl, DestVecType,
Op.getOperand(0));
5991 EVT VT =
Op.getValueType();
5994 if (isUnsupportedFloatingType(VT)) {
6002 MakeLibCallOptions CallOptions;
6004 CallOptions,
SDLoc(
Op)).first;
6015 EVT VT =
Op.getValueType();
6019 bool UseNEON = !InGPR && Subtarget->hasNEON();
6026 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
6033 if (SrcVT == MVT::f32) {
6039 }
else if (VT == MVT::f32)
6055 if (VT == MVT::f32) {
6067 if (SrcVT == MVT::f64)
6076 if (VT == MVT::f32) {
6100 EVT VT =
Op.getValueType();
6102 unsigned Depth =
Op.getConstantOperandVal(0);
6104 SDValue FrameAddr = LowerFRAMEADDR(
Op, DAG);
6123 EVT VT =
Op.getValueType();
6125 unsigned Depth =
Op.getConstantOperandVal(0);
6139 .
Case(
"sp", ARM::SP)
6154 assert(
N->getValueType(0) == MVT::i64
6155 &&
"ExpandREAD_REGISTER called for non-i64 type result.");
6158 DAG.
getVTList(MVT::i32, MVT::i32, MVT::Other),
6164 Results.push_back(Read.getOperand(0));
6198 const APInt &APIntIndex =
Index->getAPIntValue();
6200 NewIndex *= APIntIndex;
6230 EVT SrcVT =
Op.getValueType();
6231 EVT DstVT =
N->getValueType(0);
6233 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6234 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6238 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6239 (SrcVT == MVT::f16 || SrcVT == MVT::bf16))
6244 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6248 if (SrcVT == MVT::i64 && TLI.
isTypeLegal(DstVT)) {
6260 if (DstVT == MVT::i64 && TLI.
isTypeLegal(SrcVT)) {
6297 EVT VT =
Op.getValueType();
6316 SDValue LoBigShift = DAG.
getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6324 ? DAG.
getNode(Opc, dl, VT, ShOpHi,
6341 EVT VT =
Op.getValueType();
6384 DAG.
getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6436 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6464 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6494 EVT VT =
N->getValueType(0);
6495 if (VT.
isVector() && ST->hasNEON()) {
6504 if (ElemTy == MVT::i8) {
6512 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6527 if (ElemTy == MVT::i64) {
6540 if (!ST->hasV6T2Ops())
6549 EVT VT =
N->getValueType(0);
6552 assert(ST->hasNEON() &&
"Custom ctpop lowering requires NEON.");
6553 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6554 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6555 "Unexpected type for custom ctpop lowering");
6563 unsigned EltSize = 8;
6586 Op =
Op.getOperand(0);
6588 APInt SplatBits, SplatUndef;
6589 unsigned SplatBitSize;
6592 !BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6594 SplatBitSize > ElementBits)
6605 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6609 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6620 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6625 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6626 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6635 EVT VT =
N->getValueType(0);
6657 "unexpected vector shift opcode");
6659 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
6660 unsigned VShiftOpc =
6662 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
6668 EVT ShiftVT =
N->getOperand(1).getValueType();
6671 unsigned VShiftOpc =
6673 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0), NegatedCount);
6678 EVT VT =
N->getValueType(0);
6687 "Unknown shift to lower!");
6689 unsigned ShOpc =
N->getOpcode();
6690 if (ST->hasMVEIntegerOps()) {
6720 DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6734 if (ST->isThumb1Only())
6739 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6755 bool Invert =
false;
6762 EVT VT =
Op.getValueType();
6770 assert(ST->hasMVEIntegerOps() &&
6771 "No hardware support for integer vector comparison!");
6773 if (
Op.getValueType().getVectorElementType() != MVT::i1)
6798 Merged = DAG.
getNOT(dl, Merged, CmpVT);
6808 switch (SetCCOpcode) {
6812 if (ST->hasMVEFloatOps()) {
6815 Invert =
true; [[fallthrough]];
6820 case ISD::SETLT: Swap =
true; [[fallthrough]];
6824 case ISD::SETLE: Swap =
true; [[fallthrough]];
6840 Result = DAG.
getNOT(dl, Result, VT);
6843 case ISD::SETUO: Invert =
true; [[fallthrough]];
6852 Result = DAG.
getNOT(dl, Result, VT);
6858 switch (SetCCOpcode) {
6861 if (ST->hasMVEIntegerOps()) {
6864 Invert =
true; [[fallthrough]];
6867 case ISD::SETLT: Swap =
true; [[fallthrough]];
6869 case ISD::SETLE: Swap =
true; [[fallthrough]];
6878 if (ST->hasNEON() && Opc ==
ARMCC::EQ) {
6894 Result = DAG.
getNOT(dl, Result, VT);
6928 Result = DAG.
getNOT(dl, Result, VT);
6940 assert(
LHS.getSimpleValueType().isInteger() &&
"SETCCCARRY is integer only.");
6970 unsigned OpCmode, Imm;
6981 switch (SplatBitSize) {
6986 assert((SplatBits & ~0xff) == 0 &&
"one byte splat value is too big");
6989 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
6994 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
6995 if ((SplatBits & ~0xff) == 0) {
7001 if ((SplatBits & ~0xff00) == 0) {
7004 Imm = SplatBits >> 8;
7014 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
7015 if ((SplatBits & ~0xff) == 0) {
7021 if ((SplatBits & ~0xff00) == 0) {
7024 Imm = SplatBits >> 8;
7027 if ((SplatBits & ~0xff0000) == 0) {
7030 Imm = SplatBits >> 16;
7033 if ((SplatBits & ~0xff000000) == 0) {
7036 Imm = SplatBits >> 24;
7043 if ((SplatBits & ~0xffff) == 0 &&
7044 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
7047 Imm = SplatBits >> 8;
7055 if ((SplatBits & ~0xffffff) == 0 &&
7056 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
7059 Imm = SplatBits >> 16;
7075 unsigned ImmMask = 1;
7077 for (
int ByteNum = 0; ByteNum < 8; ++ByteNum) {
7078 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
7080 }
else if ((SplatBits & BitMask) != 0) {
7090 unsigned Mask = (1 << BytesPerElem) - 1;
7091 unsigned NumElems = 8 / BytesPerElem;
7092 unsigned NewImm = 0;
7093 for (
unsigned ElemNum = 0; ElemNum < NumElems; ++ElemNum) {
7094 unsigned Elem = ((Imm >> ElemNum * BytesPerElem) & Mask);
7095 NewImm |= Elem << (NumElems - ElemNum - 1) * BytesPerElem;
7102 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
7116 EVT VT =
Op.getValueType();
7117 bool IsDouble = (VT == MVT::f64);
7123 if (
ST->genExecuteOnly()) {
7125 assert((!
ST->isThumb1Only() ||
ST->hasV8MBaselineOps()) &&
7126 "Unexpected architecture");
7149 if (!
ST->hasVFP3Base())
7154 if (IsDouble && !Subtarget->hasFP64())
7161 if (IsDouble || !
ST->useNEONForSinglePrecisionFP()) {
7179 if (!
ST->hasNEON() || (!IsDouble && !
ST->useNEONForSinglePrecisionFP()))
7188 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7242 unsigned ExpectedElt = Imm;
7243 for (
unsigned i = 1; i < NumElts; ++i) {
7247 if (ExpectedElt == NumElts)
7250 if (M[i] < 0)
continue;
7251 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7259 bool &ReverseVEXT,
unsigned &Imm) {
7261 ReverseVEXT =
false;
7272 unsigned ExpectedElt = Imm;
7273 for (
unsigned i = 1; i < NumElts; ++i) {
7277 if (ExpectedElt == NumElts * 2) {
7282 if (M[i] < 0)
continue;
7283 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7298 return VT == MVT::v8i8 && M.size() == 8;
7303 if (Mask.size() == Elements * 2)
7304 return Index / Elements;
7305 return Mask[
Index] == 0 ? 0 : 1;
7335 if (M.size() != NumElts && M.size() != NumElts*2)
7343 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7345 for (
unsigned j = 0; j < NumElts; j += 2) {
7346 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7347 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + NumElts + WhichResult))
7352 if (M.size() == NumElts*2)
7367 if (M.size() != NumElts && M.size() != NumElts*2)
7370 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7372 for (
unsigned j = 0; j < NumElts; j += 2) {
7373 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7374 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + WhichResult))
7379 if (M.size() == NumElts*2)
7399 if (M.size() != NumElts && M.size() != NumElts*2)
7402 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7404 for (
unsigned j = 0; j < NumElts; ++j) {
7405 if (M[i+j] >= 0 && (
unsigned) M[i+j] != 2 * j + WhichResult)
7410 if (M.size() == NumElts*2)
7429 if (M.size() != NumElts && M.size() != NumElts*2)
7432 unsigned Half = NumElts / 2;
7433 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7435 for (
unsigned j = 0; j < NumElts; j += Half) {
7436 unsigned Idx = WhichResult;
7437 for (
unsigned k = 0; k < Half; ++k) {
7438 int MIdx = M[i + j + k];
7439 if (MIdx >= 0 && (
unsigned) MIdx !=
Idx)
7446 if (M.size() == NumElts*2)
7470 if (M.size() != NumElts && M.size() != NumElts*2)
7473 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7475 unsigned Idx = WhichResult * NumElts / 2;
7476 for (
unsigned j = 0; j < NumElts; j += 2) {
7477 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7478 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx + NumElts))
7484 if (M.size() == NumElts*2)
7503 if (M.size() != NumElts && M.size() != NumElts*2)
7506 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7508 unsigned Idx = WhichResult * NumElts / 2;
7509 for (
unsigned j = 0; j < NumElts; j += 2) {
7510 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7511 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx))
7517 if (M.size() == NumElts*2)
7530 unsigned &WhichResult,
7533 if (
isVTRNMask(ShuffleMask, VT, WhichResult))
7535 if (
isVUZPMask(ShuffleMask, VT, WhichResult))
7537 if (
isVZIPMask(ShuffleMask, VT, WhichResult))
7555 if (NumElts != M.size())
7559 for (
unsigned i = 0; i != NumElts; ++i)
7560 if (M[i] >= 0 && M[i] != (
int) (NumElts - 1 - i))
7569 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7577 int Ofs = Top ? 1 : 0;
7578 int Upper = SingleSource ? 0 : NumElts;
7579 for (
int i = 0, e = NumElts / 2; i != e; ++i) {
7580 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7582 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs +
Upper)
7591 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7600 unsigned Offset = Top ? 0 : 1;
7601 unsigned N = SingleSource ? 0 : NumElts;
7602 for (
unsigned i = 0; i < NumElts; i += 2) {
7603 if (M[i] >= 0 && M[i] != (
int)i)
7605 if (M[i + 1] >= 0 && M[i + 1] != (
int)(
N + i +
Offset))
7614 if (NumElts != M.size())
7622 unsigned Off0 = rev ? NumElts / 2 : 0;
7623 unsigned Off1 = rev ? 0 : NumElts / 2;
7624 for (
unsigned i = 0; i < NumElts; i += 2) {
7625 if (M[i] >= 0 && M[i] != (
int)(Off0 + i / 2))
7627 if (M[i + 1] >= 0 && M[i + 1] != (
int)(Off1 + i / 2))
7643 if (!ST->hasMVEFloatOps())
7648 if (VT != MVT::v8f16)
7669 for (
unsigned i = 1; i < 4; i++) {
7696 if (!ST->hasMVEFloatOps())
7701 if (VT != MVT::v4f32)
7717 for (
unsigned i = 1; i < 4; i++) {
7738 if (!isa<ConstantSDNode>(
N))
7740 Val =
N->getAsZExtVal();
7742 if (ST->isThumb1Only()) {
7743 if (Val <= 255 || ~Val <= 255)
7755 EVT VT =
Op.getValueType();
7757 assert(ST->hasMVEIntegerOps() &&
"LowerBUILD_VECTOR_i1 called without MVE!");
7761 unsigned BitsPerBool;
7765 }
else if (NumElts == 4) {
7768 }
else if (NumElts == 8) {
7771 }
else if (NumElts == 16) {
7780 if (!isa<ConstantSDNode>(FirstOp) &&
7782 return U.get().isUndef() || U.get() == FirstOp;
7790 unsigned Bits32 = 0;
7791 for (
unsigned i = 0; i < NumElts; ++i) {
7793 if (!isa<ConstantSDNode>(V) && !V.isUndef())
7795 bool BitSet = V.isUndef() ?
false : V->getAsZExtVal();
7797 Bits32 |= BoolMask << (i * BitsPerBool);
7803 for (
unsigned i = 0; i < NumElts; ++i) {
7805 if (isa<ConstantSDNode>(V) || V.isUndef())
7816 if (!ST->hasMVEIntegerOps())
7820 EVT VT =
Op.getValueType();
7830 if (
N != 1 &&
N != 2 &&
N != 4 &&
N != 8)
7834 for (
unsigned I = 2;
I < NumElts;
I++) {
7850 switch (
N->getOpcode()) {
7859 return N->getOperand(1).getNode() ==
Op;
7861 switch (
N->getConstantOperandVal(0)) {
7862 case Intrinsic::arm_mve_add_predicated:
7863 case Intrinsic::arm_mve_mul_predicated:
7864 case Intrinsic::arm_mve_qadd_predicated:
7865 case Intrinsic::arm_mve_vhadd:
7866 case Intrinsic::arm_mve_hadd_predicated:
7867 case Intrinsic::arm_mve_vqdmulh:
7868 case Intrinsic::arm_mve_qdmulh_predicated:
7869 case Intrinsic::arm_mve_vqrdmulh:
7870 case Intrinsic::arm_mve_qrdmulh_predicated:
7871 case Intrinsic::arm_mve_vqdmull:
7872 case Intrinsic::arm_mve_vqdmull_predicated:
7874 case Intrinsic::arm_mve_sub_predicated:
7875 case Intrinsic::arm_mve_qsub_predicated:
7876 case Intrinsic::arm_mve_vhsub:
7877 case Intrinsic::arm_mve_hsub_predicated:
7878 return N->getOperand(2).getNode() ==
Op;
7893 EVT VT =
Op.getValueType();
7901 APInt SplatBits, SplatUndef;
7902 unsigned SplatBitSize;
7904 if (BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7911 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7913 [BVN](
const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
7914 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7915 : SplatBitSize == 16 ? MVT::v8i16
7922 if ((
ST->hasNEON() && SplatBitSize <= 64) ||
7923 (
ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7928 SplatBitSize, DAG, dl, VmovVT, VT,
VMOVModImm);
7936 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7938 NegatedImm, SplatUndef.
getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
7946 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7956 if (
ST->hasMVEIntegerOps() &&
7957 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
7958 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7959 : SplatBitSize == 16 ? MVT::v8i16
7976 bool isOnlyLowElement =
true;
7977 bool usesOnlyOneValue =
true;
7978 bool hasDominantValue =
false;
7985 for (
unsigned i = 0; i < NumElts; ++i) {
7990 isOnlyLowElement =
false;
7991 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
7994 ValueCounts.
insert(std::make_pair(V, 0));
7995 unsigned &Count = ValueCounts[
V];
7998 if (++Count > (NumElts / 2)) {
7999 hasDominantValue =
true;
8003 if (ValueCounts.
size() != 1)
8004 usesOnlyOneValue =
false;
8005 if (!
Value.getNode() && !ValueCounts.
empty())
8008 if (ValueCounts.
empty())
8020 if (hasDominantValue && EltSize <= 32) {
8031 (constIndex = dyn_cast<ConstantSDNode>(
Value->getOperand(1)))) {
8036 if (VT !=
Value->getOperand(0).getValueType()) {
8049 if (!usesOnlyOneValue) {
8052 for (
unsigned I = 0;
I < NumElts; ++
I) {
8067 assert(FVT == MVT::f32 || FVT == MVT::f16);
8068 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
8069 for (
unsigned i = 0; i < NumElts; ++i)
8074 Val = LowerBUILD_VECTOR(Val, DAG, ST);
8078 if (usesOnlyOneValue) {
8105 if (
ST->hasNEON() && VT.
is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
8125 if (EltSize >= 32) {
8131 for (
unsigned i = 0; i < NumElts; ++i)
8145 for (
unsigned i = 0 ; i < NumElts; ++i) {
8164 EVT VT =
Op.getValueType();
8167 struct ShuffleSourceInfo {
8169 unsigned MinElt = std::numeric_limits<unsigned>::max();
8170 unsigned MaxElt = 0;
8180 int WindowScale = 1;
8182 ShuffleSourceInfo(
SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
8190 for (
unsigned i = 0; i < NumElts; ++i) {
8198 }
else if (!isa<ConstantSDNode>(
V.getOperand(1))) {
8205 SDValue SourceVec =
V.getOperand(0);
8207 if (Source == Sources.
end())
8211 unsigned EltNo =
V.getConstantOperandVal(1);
8218 if (Sources.
size() > 2)
8224 for (
auto &Source : Sources) {
8225 EVT SrcEltTy =
Source.Vec.getValueType().getVectorElementType();
8226 if (SrcEltTy.
bitsLT(SmallestEltTy))
8227 SmallestEltTy = SrcEltTy;
8229 unsigned ResMultiplier =
8237 for (
auto &Src : Sources) {
8238 EVT SrcVT = Src.ShuffleVec.getValueType();
8242 if (SrcVTSize == VTSize)
8251 if (SrcVTSize < VTSize) {
8252 if (2 * SrcVTSize != VTSize)
8258 DAG.
getUNDEF(Src.ShuffleVec.getValueType()));
8262 if (SrcVTSize != 2 * VTSize)
8265 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8270 if (Src.MinElt >= NumSrcElts) {
8275 Src.WindowBase = -NumSrcElts;
8276 }
else if (Src.MaxElt < NumSrcElts) {
8293 Src.WindowBase = -Src.MinElt;
8300 for (
auto &Src : Sources) {
8301 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8302 if (SrcEltTy == SmallestEltTy)
8307 Src.WindowBase *= Src.WindowScale;
8313 assert(Src.ShuffleVec.getValueType() == ShuffleVT););
8320 if (Entry.isUndef())
8323 auto Src =
llvm::find(Sources, Entry.getOperand(0));
8324 int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue();
8329 EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
8332 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8336 int *LaneMask = &
Mask[i * ResMultiplier];
8338 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8339 ExtractBase += NumElts * (Src - Sources.begin());
8340 for (
int j = 0;
j < LanesDefined; ++
j)
8341 LaneMask[j] = ExtractBase + j;
8347 assert(Sources.size() <= 2 &&
"Too many sources!");
8350 for (
unsigned i = 0; i < Sources.size(); ++i)
8379 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8399 unsigned PFIndexes[4];
8400 for (
unsigned i = 0; i != 4; ++i) {
8404 PFIndexes[i] = M[i];
8408 unsigned PFTableIndex =
8409 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8411 unsigned Cost = (PFEntry >> 30);
8417 bool ReverseVEXT, isV_UNDEF;
8418 unsigned Imm, WhichResult;
8421 if (EltSize >= 32 ||
8428 else if (Subtarget->hasNEON() &&
8433 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8436 else if (Subtarget->hasMVEIntegerOps() &&
8440 else if (Subtarget->hasMVEIntegerOps() &&
8454 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8455 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8456 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8459 if (LHSID == (1*9+2)*9+3)
return LHS;
8460 assert(LHSID == ((4*9+5)*9+6)*9+7 &&
"Illegal OP_COPY!");
8517 for (
int I : ShuffleMask)
8520 if (V2.getNode()->isUndef())
8530 EVT VT =
Op.getValueType();
8532 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8533 "Expect an v8i16/v16i8 type");
8539 std::vector<int> NewMask;
8543 NewMask.push_back(i);
8583 if (VT != MVT::v16i1)
8599 EVT VT =
Op.getValueType();
8603 assert(ST->hasMVEIntegerOps() &&
8604 "No support for vector shuffle of boolean predicates");
8630 "Expected identical vector type in expanded i1 shuffle!");
8634 PredAsVector2, ShuffleMask);
8639 if (VT == MVT::v2i1) {
8656 EVT VT =
Op.getValueType();
8660 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8661 "Unexpected vector type");
8663 int QuarterSize = NumElts / 4;
8672 for (
int i = 0; i <
Length; i++) {
8673 if (ShuffleMask[Start + i] >= 0) {
8674 if (ShuffleMask[Start + i] %
Length != i)
8676 MovIdx = ShuffleMask[Start + i] /
Length;
8684 for (
int i = 1; i <
Length; i++) {
8685 if (ShuffleMask[Start + i] >= 0 &&
8686 (ShuffleMask[Start + i] /
Length != MovIdx ||
8687 ShuffleMask[Start + i] %
Length != i))
8693 for (
int Part = 0; Part < 4; ++Part) {
8695 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8699 Input =
Op->getOperand(1);
8709 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8714 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8716 for (
int Part = 0; Part < 4; ++Part)
8717 for (
int i = 0; i < QuarterSize; i++)
8719 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8721 VT, dl,
Op->getOperand(0),
Op->getOperand(1), NewShuffleMask);
8724 for (
int Part = 0; Part < 4; ++Part)
8740 EVT VT =
Op.getValueType();
8752 for (
int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8756 if (Mask[i] != i + BaseOffset) {
8757 if (OffElement == -1)
8763 return NonUndef > 2 && OffElement != -1;
8767 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8769 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8780 ShuffleMask[OffElement] < (
int)NumElts ? V1 : V2,
8791 EVT VT =
Op.getValueType();
8795 if (ST->hasMVEIntegerOps() && EltSize == 1)
8806 if (EltSize <= 32) {
8810 if (Lane == -1) Lane = 0;
8821 bool IsScalarToVector =
true;
8824 IsScalarToVector =
false;
8827 if (IsScalarToVector)
8834 bool ReverseVEXT =
false;
8836 if (ST->hasNEON() &&
isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
8860 unsigned WhichResult = 0;
8861 bool isV_UNDEF =
false;
8862 if (ST->hasNEON()) {
8864 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8871 if (ST->hasMVEIntegerOps()) {
8906 }) &&
"Unexpected shuffle index into UNDEF operand!");
8909 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
8912 assert((WhichResult == 0) &&
8913 "In-place shuffle of concat can only have one result!");
8922 if (ST->hasMVEIntegerOps() && EltSize <= 32) {
8926 for (
bool Top : {
false,
true}) {
8927 for (
bool SingleSource : {
false,
true}) {
8928 if (
isTruncMask(ShuffleMask, VT, Top, SingleSource)) {
8933 SingleSource ? V1 : V2);
8949 unsigned PFIndexes[4];
8950 for (
unsigned i = 0; i != 4; ++i) {
8951 if (ShuffleMask[i] < 0)
8954 PFIndexes[i] = ShuffleMask[i];
8958 unsigned PFTableIndex =
8959 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8961 unsigned Cost = (PFEntry >> 30);
8967 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8968 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8978 if (EltSize >= 32) {
8986 for (
unsigned i = 0; i < NumElts; ++i) {
8987 if (ShuffleMask[i] < 0)
8991 ShuffleMask[i] < (
int)NumElts ? V1 : V2,
8999 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
9003 if (ST->hasNEON() && VT == MVT::v8i8)
9007 if (ST->hasMVEIntegerOps())
9016 EVT VecVT =
Op.getOperand(0).getValueType();
9019 assert(ST->hasMVEIntegerOps() &&
9020 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9024 unsigned Lane =
Op.getConstantOperandVal(2);
9025 unsigned LaneWidth =
9027 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
9039 if (!isa<ConstantSDNode>(Lane))
9045 if (Subtarget->hasMVEIntegerOps() &&
9046 Op.getValueType().getScalarSizeInBits() == 1)
9070 IVecIn, IElt, Lane);
9079 EVT VecVT =
Op.getOperand(0).getValueType();
9082 assert(ST->hasMVEIntegerOps() &&
9083 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9087 unsigned Lane =
Op.getConstantOperandVal(1);
9088 unsigned LaneWidth =
9099 if (!isa<ConstantSDNode>(Lane))
9119 assert(
Op.getValueType().getScalarSizeInBits() == 1 &&
9120 "Unexpected custom CONCAT_VECTORS lowering");
9122 "Unexpected custom CONCAT_VECTORS lowering");
9123 assert(ST->hasMVEIntegerOps() &&
9124 "CONCAT_VECTORS lowering only supported for MVE");
9128 EVT Op2VT = V2.getValueType();
9129 assert(Op1VT == Op2VT &&
"Operand types don't match!");
9130 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
9131 "Unexpected i1 concat operations!");
9144 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
9158 auto ExtractInto = [&DAG, &dl](
SDValue NewV,
SDValue ConVec,
unsigned &j) {
9159 EVT NewVT = NewV.getValueType();
9160 EVT ConcatVT = ConVec.getValueType();
9161 unsigned ExtScale = 1;
9162 if (NewVT == MVT::v2f64) {
9176 ConVec = ExtractInto(NewV1, ConVec, j);
9177 ConVec = ExtractInto(NewV2, ConVec, j);
9187 while (ConcatOps.
size() > 1) {
9188 for (
unsigned I = 0, E = ConcatOps.
size();
I != E;
I += 2) {
9191 ConcatOps[
I / 2] = ConcatPair(V1, V2);
9195 return ConcatOps[0];
9200 EVT VT =
Op->getValueType(0);
9207 "unexpected CONCAT_VECTORS");
9228 EVT VT =
Op.getValueType();
9231 unsigned Index = V2->getAsZExtVal();
9234 "Unexpected custom EXTRACT_SUBVECTOR lowering");
9235 assert(ST->hasMVEIntegerOps() &&
9236 "EXTRACT_SUBVECTOR lowering only supported for MVE");
9246 EVT SubVT = MVT::v4i32;
9248 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j += 2) {
9263 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j++) {
9279 assert(ST->hasMVEIntegerOps() &&
"Expected MVE!");
9280 EVT VT =
N->getValueType(0);
9281 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9282 "Expected a vector i1 type!");
9284 EVT FromVT =
Op.getValueType();
9295 if (!Subtarget->hasMVEIntegerOps())
9298 EVT ToVT =
N->getValueType(0);
9341 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9343 EVT FromVT =
N->getOperand(0).getValueType();
9344 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9355 if (!Subtarget->hasMVEIntegerOps())
9360 EVT ToVT =
N->getValueType(0);
9361 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9364 EVT FromVT =
Op.getValueType();
9365 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9376 SDValue Ext1 = Ext.getValue(1);
9379 Ext = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext);
9380 Ext1 = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext1);
9392 EVT VT =
N->getValueType(0);
9394 SDNode *BVN =
N->getOperand(0).getNode();
9399 unsigned HiElt = 1 - LoElt;
9404 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9420 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
9421 SDNode *Elt =
N->getOperand(i).getNode();
9424 unsigned HalfSize = EltSize / 2;
9426 if (!
isIntN(HalfSize,
C->getSExtValue()))
9429 if (!
isUIntN(HalfSize,
C->getZExtValue()))
9468 switch (OrigSimpleTy) {
9484 unsigned ExtOpcode) {
9507 if (ExtendedTy == LD->getMemoryVT())
9508 return DAG.
getLoad(LD->getMemoryVT(),
SDLoc(LD), LD->getChain(),
9509 LD->getBasePtr(), LD->getPointerInfo(), LD->getAlign(),
9510 LD->getMemOperand()->getFlags());
9516 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
9517 LD->getMemoryVT(), LD->getAlign(),
9518 LD->getMemOperand()->getFlags());
9531 N->getOperand(0)->getValueType(0),
9537 "Expected extending load");
9543 DAG.
getNode(Opcode,
SDLoc(newLoad), LD->getValueType(0), newLoad);
9552 SDNode *BVN =
N->getOperand(0).getNode();
9554 BVN->
getValueType(0) == MVT::v4i32 &&
"expected v4i32 BUILD_VECTOR");
9562 EVT VT =
N->getValueType(0);
9568 for (
unsigned i = 0; i != NumElts; ++i) {
9569 const APInt &CInt =
N->getConstantOperandAPInt(i);
9578 unsigned Opcode =
N->getOpcode();
9580 SDNode *N0 =
N->getOperand(0).getNode();
9581 SDNode *N1 =
N->getOperand(1).getNode();
9589 unsigned Opcode =
N->getOpcode();
9591 SDNode *N0 =
N->getOperand(0).getNode();
9592 SDNode *N1 =
N->getOperand(1).getNode();
9602 EVT VT =
Op.getValueType();
9604 "unexpected type for custom-lowering ISD::MUL");
9605 SDNode *N0 =
Op.getOperand(0).getNode();
9606 SDNode *N1 =
Op.getOperand(1).getNode();
9607 unsigned NewOpc = 0;
9611 if (isN0SExt && isN1SExt)
9616 if (isN0ZExt && isN1ZExt)
9618 else if (isN1SExt || isN1ZExt) {
9635 if (VT == MVT::v2i64)
9652 "unexpected types for extended operands to VMULL");
9653 return DAG.
getNode(NewOpc,
DL, VT, Op0, Op1);
9688 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9722 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9725 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9746 EVT VT =
Op.getValueType();
9747 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9748 "unexpected type for custom-lowering ISD::SDIV");
9755 if (VT == MVT::v8i8) {
9783 EVT VT =
Op.getValueType();
9784 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9785 "unexpected type for custom-lowering ISD::UDIV");
9792 if (VT == MVT::v8i8) {
9831 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9834 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9838 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9859 EVT VT =
N->getValueType(0);
9873 Op.getOperand(1), Carry);
9887 Op.getOperand(1), Carry);
9920 bool ShouldUseSRet = Subtarget->
isAPCS_ABI();
9922 if (ShouldUseSRet) {
9932 Entry.IsSExt =
false;
9933 Entry.IsZExt =
false;
9934 Entry.IsSRet =
true;
9935 Args.push_back(Entry);
9942 Entry.IsSExt =
false;
9943 Entry.IsZExt =
false;
9944 Args.push_back(Entry);
9947 (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
9957 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
9960 return CallResult.first;
9979 EVT VT =
Op.getValueType();
9980 assert((VT == MVT::i32 || VT == MVT::i64) &&
9981 "unexpected type for custom lowering DIV");
9987 const char *
Name =
nullptr;
9989 Name = (VT == MVT::i32) ?
"__rt_sdiv" :
"__rt_sdiv64";
9991 Name = (VT == MVT::i32) ?
"__rt_udiv" :
"__rt_udiv64";
9997 for (
auto AI : {1, 0}) {
9999 Arg.Node =
Op.getOperand(AI);
10000 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.
getContext());
10001 Args.push_back(Arg);
10004 CallLoweringInfo CLI(DAG);
10008 ES, std::move(Args));
10018ARMTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
10026 const bool MinSize =
ST.hasMinSize();
10027 const bool HasDivide =
ST.isThumb() ?
ST.hasDivideInThumbMode()
10028 :
ST.hasDivideInARMMode();
10032 if (
N->getOperand(0).getValueType().isVector())
10037 if (!(MinSize && HasDivide))
10050 if (Divisor.
sgt(128))
10058 assert(
Op.getValueType() == MVT::i32 &&
10059 "unexpected type for custom lowering DIV");
10065 return LowerWindowsDIVLibCall(
Op, DAG,
Signed, DBZCHK);
10071 if (
N->getValueType(0) == MVT::i32)
10079void ARMTargetLowering::ExpandDIV_Windows(
10085 assert(
Op.getValueType() == MVT::i64 &&
10086 "unexpected type for custom lowering DIV");
10103 EVT MemVT = LD->getMemoryVT();
10104 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10105 MemVT == MVT::v16i1) &&
10106 "Expected a predicate type!");
10107 assert(MemVT ==
Op.getValueType());
10109 "Expected a non-extending load");
10110 assert(LD->isUnindexed() &&
"Expected a unindexed load");
10124 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(),
10126 LD->getMemOperand());
10133 if (MemVT != MVT::v16i1)
10142 EVT MemVT =
LD->getMemoryVT();
10143 assert(
LD->isUnindexed() &&
"Loads should be unindexed at this point.");
10145 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10151 {LD->getChain(), LD->getBasePtr()}, MemVT,
LD->getMemOperand());
10161 EVT MemVT = ST->getMemoryVT();
10162 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10163 MemVT == MVT::v16i1) &&
10164 "Expected a predicate type!");
10165 assert(MemVT == ST->getValue().getValueType());
10166 assert(!ST->isTruncatingStore() &&
"Expected a non-extending store");
10167 assert(ST->isUnindexed() &&
"Expected a unindexed store");
10172 SDValue Build = ST->getValue();
10173 if (MemVT != MVT::v16i1) {
10192 ST->getChain(), dl, GRP, ST->getBasePtr(),
10194 ST->getMemOperand());
10200 EVT MemVT = ST->getMemoryVT();
10201 assert(ST->isUnindexed() &&
"Stores should be unindexed at this point.");
10203 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10219 {ST->getChain(), Lo, Hi, ST->getBasePtr()},
10220 MemVT, ST->getMemOperand());
10221 }
else if (Subtarget->hasMVEIntegerOps() &&
10222 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10223 MemVT == MVT::v16i1))) {
10238 MVT VT =
Op.getSimpleValueType();
10240 SDValue PassThru =
N->getPassThru();
10251 VT, dl,
N->getChain(),
N->getBasePtr(),
N->getOffset(), Mask, ZeroVec,
10252 N->getMemoryVT(),
N->getMemOperand(),
N->getAddressingMode(),
10253 N->getExtensionType(),
N->isExpandingLoad());
10258 if (!PassThru.
isUndef() && !PassThruIsCastZero)
10265 if (!ST->hasMVEIntegerOps())
10269 unsigned BaseOpcode = 0;
10270 switch (
Op->getOpcode()) {
10286 unsigned NumActiveLanes = NumElts;
10288 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10289 NumActiveLanes == 2) &&
10290 "Only expected a power 2 vector size");
10294 while (NumActiveLanes > 4) {
10297 Op0 = DAG.
getNode(BaseOpcode, dl, VT, Op0, Rev);
10298 NumActiveLanes /= 2;
10302 if (NumActiveLanes == 4) {
10312 SDValue Res0 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10313 SDValue Res1 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext2, Ext3,
Op->getFlags());
10314 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res0, Res1,
Op->getFlags());
10320 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10324 if (EltVT !=
Op->getValueType(0))
10331 if (!ST->hasMVEFloatOps())
10338 if (!ST->hasNEON())
10346 unsigned PairwiseIntrinsic = 0;
10347 switch (
Op->getOpcode()) {
10351 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
10354 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
10357 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
10360 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10366 unsigned NumActiveLanes = NumElts;
10368 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10369 NumActiveLanes == 2) &&
10370 "Only expected a power 2 vector size");
10376 VT =
Lo.getValueType();
10378 NumActiveLanes /= 2;
10382 while (NumActiveLanes > 1) {
10384 NumActiveLanes /= 2;
10391 if (EltVT !=
Op.getValueType()) {
10392 unsigned Extend = 0;
10393 switch (
Op->getOpcode()) {
10405 Res = DAG.
getNode(Extend, dl,
Op.getValueType(), Res);
10427 SDValue Ops[] = {
N->getOperand(0),
10437 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
10444 SDLoc dl(V.getNode());
10445 auto [VLo, VHi] = DAG.
SplitScalar(V, dl, MVT::i32, MVT::i32);
10453 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
10455 DAG.
getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
10461 assert(
N->getValueType(0) == MVT::i64 &&
10462 "AtomicCmpSwap on types less than 64 should be legal");
10463 SDValue Ops[] = {
N->getOperand(1),
10468 ARM::CMP_SWAP_64,
SDLoc(
N),
10469 DAG.
getVTList(MVT::Untyped, MVT::i32, MVT::Other), Ops);
10488 EVT VT =
Op.getValueType();
10497 if (isUnsupportedFloatingType(
LHS.getValueType())) {
10499 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS, Chain, IsSignaling);
10500 if (!
RHS.getNode()) {
10521 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10522 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, CCR, Cmp, DAG);
10524 ARMcc = DAG.
getConstant(CondCode2, dl, MVT::i32);
10525 Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10526 Result = getCMOV(dl, VT, Result, True, ARMcc, CCR, Cmp, DAG);
10542 switch (
Op.getOpcode()) {
10574 case ISD::BITCAST:
return ExpandBITCAST(
Op.getNode(), DAG, Subtarget);
10578 case ISD::SREM:
return LowerREM(
Op.getNode(), DAG);
10579 case ISD::UREM:
return LowerREM(
Op.getNode(), DAG);
10601 return LowerSET_FPMODE(
Op, DAG);
10603 return LowerRESET_FPMODE(
Op, DAG);
10607 return LowerDIV_Windows(
Op, DAG,
true);
10611 return LowerDIV_Windows(
Op, DAG,
false);
10618 return LowerSignedALUO(
Op, DAG);
10621 return LowerUnsignedALUO(
Op, DAG);
10655 return LowerDYNAMIC_STACKALLOC(
Op, DAG);
10664 return LowerSPONENTRY(
Op, DAG);
10671 unsigned IntNo =
N->getConstantOperandVal(0);
10673 if (IntNo == Intrinsic::arm_smlald)
10675 else if (IntNo == Intrinsic::arm_smlaldx)
10677 else if (IntNo == Intrinsic::arm_smlsld)
10679 else if (IntNo == Intrinsic::arm_smlsldx)
10686 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(3), dl, MVT::i32, MVT::i32);
10690 N->getOperand(1),
N->getOperand(2),
10702 switch (
N->getOpcode()) {
10709 Res = ExpandBITCAST(
N, DAG, Subtarget);
10718 Res = LowerREM(
N, DAG);
10722 Res = LowerDivRem(
SDValue(
N, 0), DAG);
10776 "ROPI/RWPI not currently supported with SjLj");
10785 bool isThumb = Subtarget->isThumb();
10786 bool isThumb2 = Subtarget->
isThumb2();
10789 unsigned PCAdj = (
isThumb || isThumb2) ? 4 : 8;
10795 : &ARM::GPRRegClass;
10813 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10819 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10825 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10843 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10848 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10853 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10858 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
10864 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
10879 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10885 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10909 : &ARM::GPRnopcRegClass;
10914 unsigned MaxCSNum = 0;
10922 if (!II.isEHLabel())
10926 if (!MF->hasCallSiteLandingPad(
Sym))
continue;
10929 for (
unsigned Idx : CallSiteIdxs) {
10930 CallSiteNumToLPad[
Idx].push_back(&BB);
10931 MaxCSNum = std::max(MaxCSNum,
Idx);
10938 std::vector<MachineBasicBlock*> LPadList;
10940 LPadList.reserve(CallSiteNumToLPad.
size());
10941 for (
unsigned I = 1;
I <= MaxCSNum; ++
I) {
10944 LPadList.push_back(
MBB);
10949 assert(!LPadList.empty() &&
10950 "No landing pad destinations for the dispatch jump table!");
10964 unsigned trap_opcode;
10965 if (Subtarget->isThumb())
10966 trap_opcode = ARM::tTRAP;
10968 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP;
10977 MF->insert(MF->end(), DispatchBB);
10978 MF->insert(MF->end(), DispContBB);
10979 MF->insert(MF->end(), TrapBB);
10983 SetupEntryBlockForSjLj(
MI,
MBB, DispatchBB, FI);
10990 MIB =
BuildMI(DispatchBB, dl,
TII->get(ARM::Int_eh_sjlj_dispatchsetup));
11002 unsigned NumLPads = LPadList.size();
11004 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11005 BuildMI(DispatchBB, dl,
TII->get(ARM::t2LDRi12), NewVReg1)
11011 if (NumLPads < 256) {
11012 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPri))
11014 .
addImm(LPadList.size())
11017 Register VReg1 =
MRI->createVirtualRegister(TRC);
11018 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVi16), VReg1)
11019 .
addImm(NumLPads & 0xFFFF)
11022 unsigned VReg2 = VReg1;
11023 if ((NumLPads & 0xFFFF0000) != 0) {
11024 VReg2 =
MRI->createVirtualRegister(TRC);
11025 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVTi16), VReg2)
11031 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPrr))
11037 BuildMI(DispatchBB, dl,
TII->get(ARM::t2Bcc))
11042 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11043 BuildMI(DispContBB, dl,
TII->get(ARM::t2LEApcrelJT), NewVReg3)
11047 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11048 BuildMI(DispContBB, dl,
TII->get(ARM::t2ADDrs), NewVReg4)
11055 BuildMI(DispContBB, dl,
TII->get(ARM::t2BR_JT))
11059 }
else if (Subtarget->isThumb()) {
11060 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11061 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRspi), NewVReg1)
11067 if (NumLPads < 256) {
11068 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPi8))
11078 Align Alignment = MF->getDataLayout().getPrefTypeAlign(
Int32Ty);
11081 Register VReg1 =
MRI->createVirtualRegister(TRC);
11082 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRpci))
11086 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPr))
11092 BuildMI(DispatchBB, dl,
TII->get(ARM::tBcc))
11097 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
11098 BuildMI(DispContBB, dl,
TII->get(ARM::tLSLri), NewVReg2)
11104 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11105 BuildMI(DispContBB, dl,
TII->get(ARM::tLEApcrelJT), NewVReg3)
11109 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11110 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg4)
11120 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11121 BuildMI(DispContBB, dl,
TII->get(ARM::tLDRi), NewVReg5)
11127 unsigned NewVReg6 = NewVReg5;
11128 if (IsPositionIndependent) {
11129 NewVReg6 =
MRI->createVirtualRegister(TRC);
11130 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg6)
11137 BuildMI(DispContBB, dl,
TII->get(ARM::tBR_JTr))
11141 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11142 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRi12), NewVReg1)
11148 if (NumLPads < 256) {
11149 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPri))
11153 }
else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) {
11154 Register VReg1 =
MRI->createVirtualRegister(TRC);
11155 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVi16), VReg1)
11156 .
addImm(NumLPads & 0xFFFF)
11159 unsigned VReg2 = VReg1;
11160 if ((NumLPads & 0xFFFF0000) != 0) {
11161 VReg2 =
MRI->createVirtualRegister(TRC);
11162 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVTi16), VReg2)
11168 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11178 Align Alignment = MF->getDataLayout().getPrefTypeAlign(
Int32Ty);
11181 Register VReg1 =
MRI->createVirtualRegister(TRC);
11182 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRcp))
11187 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11198 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11199 BuildMI(DispContBB, dl,
TII->get(ARM::MOVsi), NewVReg3)
11204 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11205 BuildMI(DispContBB, dl,
TII->get(ARM::LEApcrelJT), NewVReg4)
11212 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11213 BuildMI(DispContBB, dl,
TII->get(ARM::LDRrs), NewVReg5)
11220 if (IsPositionIndependent) {
11221 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTadd))
11226 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTr))
11235 if (SeenMBBs.
insert(CurMBB).second)
11247 while (!Successors.empty()) {
11256 BB->normalizeSuccProbs();
11263 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
11264 if (!II->isCall())
continue;
11268 OI = II->operands_begin(), OE = II->operands_end();
11270 if (!OI->isReg())
continue;
11271 DefRegs[OI->getReg()] =
true;
11276 for (
unsigned i = 0; SavedRegs[i] != 0; ++i) {
11277 unsigned Reg = SavedRegs[i];
11279 !ARM::tGPRRegClass.contains(Reg) &&
11280 !ARM::hGPRRegClass.contains(Reg))
11282 if (Subtarget->
isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11284 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11297 MBBLPad->setIsEHPad(
false);
11300 MI.eraseFromParent();
11313static unsigned getLdOpcode(
unsigned LdSize,
bool IsThumb1,
bool IsThumb2) {
11315 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11316 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11318 return LdSize == 4 ? ARM::tLDRi
11319 : LdSize == 2 ? ARM::tLDRHi
11320 : LdSize == 1 ? ARM::tLDRBi : 0;
11322 return LdSize == 4 ? ARM::t2LDR_POST
11323 : LdSize == 2 ? ARM::t2LDRH_POST
11324 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11325 return LdSize == 4 ? ARM::LDR_POST_IMM
11326 : LdSize == 2 ? ARM::LDRH_POST
11327 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11332static unsigned getStOpcode(
unsigned StSize,
bool IsThumb1,
bool IsThumb2) {
11334 return StSize == 16 ? ARM::VST1q32wb_fixed
11335 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11337 return StSize == 4 ? ARM::tSTRi
11338 : StSize == 2 ? ARM::tSTRHi
11339 : StSize == 1 ? ARM::tSTRBi : 0;
11341 return StSize == 4 ? ARM::t2STR_POST
11342 : StSize == 2 ? ARM::t2STRH_POST
11343 : StSize == 1 ? ARM::t2STRB_POST : 0;
11344 return StSize == 4 ? ARM::STR_POST_IMM
11345 : StSize == 2 ? ARM::STRH_POST
11346 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11353 unsigned LdSize,
unsigned Data,
unsigned AddrIn,
11354 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11355 unsigned LdOpc =
getLdOpcode(LdSize, IsThumb1, IsThumb2);
11356 assert(LdOpc != 0 &&
"Should have a load opcode");
11363 }
else if (IsThumb1) {
11369 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11374 }
else if (IsThumb2) {
11394 unsigned StSize,
unsigned Data,
unsigned AddrIn,
11395 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11396 unsigned StOpc =
getStOpcode(StSize, IsThumb1, IsThumb2);
11397 assert(StOpc != 0 &&
"Should have a store opcode");
11399 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11404 }
else if (IsThumb1) {
11411 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11416 }
else if (IsThumb2) {
11417 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11423 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11444 unsigned SizeVal =
MI.getOperand(2).getImm();
11445 unsigned Alignment =
MI.getOperand(3).getImm();
11450 unsigned UnitSize = 0;
11455 bool IsThumb2 = Subtarget->
isThumb2();
11456 bool IsThumb = Subtarget->isThumb();
11458 if (Alignment & 1) {
11460 }
else if (Alignment & 2) {
11465 Subtarget->hasNEON()) {
11466 if ((Alignment % 16 == 0) && SizeVal >= 16)
11468 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11477 bool IsNeon = UnitSize >= 8;
11478 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11480 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11481 : UnitSize == 8 ? &ARM::DPRRegClass
11484 unsigned BytesLeft = SizeVal % UnitSize;
11485 unsigned LoopSize = SizeVal - BytesLeft;
11487 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11491 unsigned srcIn = src;
11492 unsigned destIn = dest;
11493 for (
unsigned i = 0; i < LoopSize; i+=UnitSize) {
11494 Register srcOut =
MRI.createVirtualRegister(TRC);
11495 Register destOut =
MRI.createVirtualRegister(TRC);
11496 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11498 IsThumb1, IsThumb2);
11500 IsThumb1, IsThumb2);
11508 for (
unsigned i = 0; i < BytesLeft; i++) {
11509 Register srcOut =
MRI.createVirtualRegister(TRC);
11510 Register destOut =
MRI.createVirtualRegister(TRC);
11511 Register scratch =
MRI.createVirtualRegister(TRC);
11513 IsThumb1, IsThumb2);
11515 IsThumb1, IsThumb2);
11519 MI.eraseFromParent();
11545 MF->
insert(It, loopMBB);
11546 MF->
insert(It, exitMBB);
11549 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
11559 Register varEnd =
MRI.createVirtualRegister(TRC);
11561 BuildMI(BB, dl,
TII->get(IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11564 }
else if (Subtarget->genExecuteOnly()) {
11565 assert(IsThumb &&
"Non-thumb expected to have used movt");
11601 Register varLoop =
MRI.createVirtualRegister(TRC);
11602 Register varPhi =
MRI.createVirtualRegister(TRC);
11603 Register srcLoop =
MRI.createVirtualRegister(TRC);
11604 Register srcPhi =
MRI.createVirtualRegister(TRC);
11605 Register destLoop =
MRI.createVirtualRegister(TRC);
11606 Register destPhi =
MRI.createVirtualRegister(TRC);
11614 BuildMI(BB, dl,
TII->get(ARM::PHI), destPhi)
11620 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11622 IsThumb1, IsThumb2);
11624 IsThumb1, IsThumb2);
11628 BuildMI(*BB, BB->
end(), dl,
TII->get(ARM::tSUBi8), varLoop)
11636 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
11645 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11654 auto StartOfExit = exitMBB->
begin();
11658 unsigned srcIn = srcLoop;
11659 unsigned destIn = destLoop;
11660 for (
unsigned i = 0; i < BytesLeft; i++) {
11661 Register srcOut =
MRI.createVirtualRegister(TRC);
11662 Register destOut =
MRI.createVirtualRegister(TRC);
11663 Register scratch =
MRI.createVirtualRegister(TRC);
11664 emitPostLd(BB, StartOfExit,
TII, dl, 1, scratch, srcIn, srcOut,
11665 IsThumb1, IsThumb2);
11666 emitPostSt(BB, StartOfExit,
TII, dl, 1, scratch, destIn, destOut,
11667 IsThumb1, IsThumb2);
11672 MI.eraseFromParent();
11684 "__chkstk is only supported on Windows");
11685 assert(Subtarget->
isThumb2() &&
"Windows on ARM requires Thumb-2 mode");
11705 switch (
TM.getCodeModel()) {
11747 MI.eraseFromParent();
11771 .
addReg(
MI.getOperand(0).getReg())
11779 MI.eraseFromParent();
11803 if (miI == BB->
end()) {
11805 if (Succ->isLiveIn(ARM::CPSR))
11811 SelectItr->addRegisterKilled(ARM::CPSR,
TRI);
11823 Register AddDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11824 BuildMI(TpEntry, Dl,
TII->get(ARM::t2ADDri), AddDestReg)
11830 Register LsrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11831 BuildMI(TpEntry, Dl,
TII->get(ARM::t2LSRri), LsrDestReg)
11837 Register TotalIterationsReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11838 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg)
11841 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopStart))
11842 .
addUse(TotalIterationsReg)
11849 return TotalIterationsReg;
11860 Register TotalIterationsReg,
bool IsMemcpy) {
11867 SrcPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11868 CurrSrcReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11869 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), SrcPhiReg)
11877 Register DestPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11878 Register CurrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11879 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), DestPhiReg)
11886 Register LoopCounterPhiReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11887 Register RemainingLoopIterationsReg =
11888 MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11889 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), LoopCounterPhiReg)
11890 .
addUse(TotalIterationsReg)
11892 .
addUse(RemainingLoopIterationsReg)
11896 Register PredCounterPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11897 Register RemainingElementsReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11898 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), PredCounterPhiReg)
11899 .
addUse(ElementCountReg)
11901 .
addUse(RemainingElementsReg)
11905 Register VccrReg =
MRI.createVirtualRegister(&ARM::VCCRRegClass);
11906 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VCTP8), VccrReg)
11907 .
addUse(PredCounterPhiReg)
11912 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2SUBri), RemainingElementsReg)
11913 .
addUse(PredCounterPhiReg)
11921 SrcValueReg =
MRI.createVirtualRegister(&ARM::MQPRRegClass);
11922 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VLDRBU8_post))
11931 SrcValueReg = OpSrcReg;
11933 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VSTRBU8_post))
11944 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg)
11945 .
addUse(LoopCounterPhiReg)
11948 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopEnd))
11949 .
addUse(RemainingLoopIterationsReg)
11962 bool isThumb2 = Subtarget->
isThumb2();
11963 switch (
MI.getOpcode()) {
11970 case ARM::tLDR_postidx: {
11974 .
add(
MI.getOperand(2))
11975 .
add(
MI.getOperand(3))
11976 .
add(
MI.getOperand(4))
11977 .
add(
MI.getOperand(0))
11979 MI.eraseFromParent();
11983 case ARM::MVE_MEMCPYLOOPINST:
11984 case ARM::MVE_MEMSETLOOPINST: {
12014 Register OpDestReg =
MI.getOperand(0).getReg();
12015 Register OpSrcReg =
MI.getOperand(1).getReg();
12016 Register OpSizeReg =
MI.getOperand(2).getReg();
12036 if (TpExit == BB) {
12038 "block containing memcpy/memset Pseudo");
12051 bool IsMemcpy =
MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
12053 OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy);
12068 MI.eraseFromParent();
12078 case ARM::t2STR_preidx:
12079 MI.setDesc(
TII->get(ARM::t2STR_PRE));
12081 case ARM::t2STRB_preidx:
12082 MI.setDesc(
TII->get(ARM::t2STRB_PRE));
12084 case ARM::t2STRH_preidx:
12085 MI.setDesc(
TII->get(ARM::t2STRH_PRE));
12088 case ARM::STRi_preidx:
12089 case ARM::STRBi_preidx: {
12090 unsigned NewOpc =
MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
12091 : ARM::STRB_PRE_IMM;
12093 unsigned Offset =
MI.getOperand(4).getImm();
12101 .
add(
MI.getOperand(0))
12102 .
add(
MI.getOperand(1))
12103 .
add(
MI.getOperand(2))
12105 .
add(
MI.getOperand(5))
12106 .
add(
MI.getOperand(6))
12108 MI.eraseFromParent();
12111 case ARM::STRr_preidx:
12112 case ARM::STRBr_preidx:
12113 case ARM::STRH_preidx: {
12115 switch (
MI.getOpcode()) {
12117 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG;
break;
12118 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG;
break;
12119 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE;
break;
12124 MI.eraseFromParent();
12128 case ARM::tMOVCCr_pseudo: {
12146 F->insert(It, copy0MBB);
12147 F->insert(It, sinkMBB);
12150 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
12156 if (!
MI.killsRegister(ARM::CPSR,
nullptr) &&
12172 .
addImm(
MI.getOperand(3).getImm())
12173 .
addReg(
MI.getOperand(4).getReg());
12188 .
addReg(
MI.getOperand(1).getReg())
12190 .
addReg(
MI.getOperand(2).getReg())
12193 MI.eraseFromParent();
12198 case ARM::BCCZi64: {
12204 bool RHSisZero =
MI.getOpcode() == ARM::BCCZi64;
12209 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12213 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12219 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12223 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12233 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12242 MI.eraseFromParent();
12246 case ARM::Int_eh_sjlj_setjmp:
12247 case ARM::Int_eh_sjlj_setjmp_nofp:
12248 case ARM::tInt_eh_sjlj_setjmp:
12249 case ARM::t2Int_eh_sjlj_setjmp:
12250 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12253 case ARM::Int_eh_sjlj_setup_dispatch:
12254 EmitSjLjDispatchBlock(
MI, BB);
12277 Fn->
insert(BBI, SinkBB);
12279 Register ABSSrcReg =
MI.getOperand(1).getReg();
12280 Register ABSDstReg =
MI.getOperand(0).getReg();
12281 bool ABSSrcKIll =
MI.getOperand(1).isKill();
12282 bool isThumb2 = Subtarget->
isThumb2();
12286 Register NewRsbDstReg =
MRI.createVirtualRegister(
12287 isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass);
12301 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12308 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).
addMBB(SinkBB)
12315 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
12324 TII->get(ARM::PHI), ABSDstReg)
12329 MI.eraseFromParent();
12334 case ARM::COPY_STRUCT_BYVAL_I32:
12336 return EmitStructByval(
MI, BB);
12337 case ARM::WIN__CHKSTK:
12338 return EmitLowered__chkstk(
MI, BB);
12339 case ARM::WIN__DBZCHK:
12340 return EmitLowered__dbzchk(
MI, BB);
12357 if (!
Node->hasAnyUseOfValue(0)) {
12358 MI.getOperand(0).setIsDead(
true);
12360 if (!
Node->hasAnyUseOfValue(1)) {
12361 MI.getOperand(1).setIsDead(
true);
12365 for (
unsigned I = 0;
I !=
MI.getOperand(4).getImm(); ++
I) {
12366 Register TmpReg =
MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
12367 : &ARM::GPRRegClass);
12374 if (
MI.getOpcode() == ARM::MEMCPY) {
12392 MCID = &
TII->get(NewOpc);
12395 MI.getDesc().getNumOperands() + 5 -
MI.getDesc().getSize()
12396 &&
"converted opcode should be the same except for cc_out"
12397 " (and, on Thumb1, pred)");
12407 MI.addOperand(
MI.getOperand(1));
12408 MI.removeOperand(1);
12412 for (
unsigned i =
MI.getNumOperands(); i--;) {
12414 if (
op.isReg() &&
op.isUse()) {
12417 MI.tieOperands(DefIdx, i);
12431 if (!
MI.hasOptionalDef() || !MCID->
operands()[ccOutIdx].isOptionalDef()) {
12432 assert(!NewOpc &&
"Optional cc_out operand required");
12437 bool definesCPSR =
false;
12438 bool deadCPSR =
false;
12439 for (
unsigned i = MCID->
getNumOperands(), e =
MI.getNumOperands(); i != e;
12443 definesCPSR =
true;
12446 MI.removeOperand(i);
12450 if (!definesCPSR) {
12451 assert(!NewOpc &&
"Optional cc_out operand required");
12454 assert(deadCPSR == !Node->hasAnyUseOfValue(1) &&
"inconsistent dead flag");
12456 assert(!
MI.getOperand(ccOutIdx).getReg() &&
12457 "expect uninitialized optional cc_out operand");
12495 switch (
N->getOpcode()) {
12496 default:
return false;
12498 CC =
N->getOperand(0);
12520 EVT VT =
N->getValueType(0);
12521 CC =
N->getOperand(0);
12522 if (
CC.getValueType() != MVT::i1 ||
CC.getOpcode() !=
ISD::SETCC)
12568 EVT VT =
N->getValueType(0);
12571 bool SwapSelectOps;
12573 NonConstantVal, DAG))
12579 OtherOp, NonConstantVal);
12585 CCOp, TrueVal, FalseVal);
12609 if (
N->getOpcode() ==
ARMISD::VTRN &&
N->getValueType(0) == MVT::v2i32)
12624 if (!
N->getValueType(0).is64BitVector())
12632 EVT VT =
N->getValueType(0);
12671 EVT VT =
N->getValueType(0);
12677 Opcode = Intrinsic::arm_neon_vpaddls;
12679 Opcode = Intrinsic::arm_neon_vpaddlu;
12707 EVT VT =
N->getValueType(0);
12722 unsigned nextIndex = 0;
12790 return DAG.
getNode(ExtOp, dl, VT, tmp);
12821 if (SRA.getOpcode() !=
ISD::SRA) {
12827 if (
auto Const = dyn_cast<ConstantSDNode>(SRA.getOperand(1))) {
12828 if (Const->getZExtValue() != 31)
12833 if (SRA.getOperand(0) !=
Mul)
12837 SDLoc dl(AddcNode);
12838 unsigned Opcode = 0;
12844 Op0 =
Mul.getOperand(0);
12845 Op1 =
Mul.getOperand(1);
12848 Op0 =
Mul.getOperand(0);
12849 Op1 =
Mul.getOperand(1).getOperand(0);
12852 Op0 =
Mul.getOperand(0).getOperand(0);
12853 Op1 =
Mul.getOperand(1);
12856 Op0 =
Mul->getOperand(0).getOperand(0);
12857 Op1 =
Mul->getOperand(1).getOperand(0);
12866 SDValue HiMLALResult(SMLAL.getNode(), 1);
12867 SDValue LoMLALResult(SMLAL.getNode(), 0);
12873 SDValue resNode(AddcNode, 0);
12902 "Expect an ADDE or SUBE");
12906 "ADDE node has the wrong inputs");
12925 "Expect ADDC with two result values. First: i32");
12945 bool IsLeftOperandMUL =
false;
12950 IsLeftOperandMUL =
true;
12961 SDValue *LowAddSub =
nullptr;
12964 if ((AddeSubeOp0 != MULOp.
getValue(1)) && (AddeSubeOp1 != MULOp.
getValue(1)))
12967 if (IsLeftOperandMUL)
12968 HiAddSub = &AddeSubeOp1;
12970 HiAddSub = &AddeSubeOp0;
12975 if (AddcSubcOp0 == MULOp.
getValue(0)) {
12976 LoMul = &AddcSubcOp0;
12977 LowAddSub = &AddcSubcOp1;
12979 if (AddcSubcOp1 == MULOp.
getValue(0)) {
12980 LoMul = &AddcSubcOp1;
12981 LowAddSub = &AddcSubcOp0;
12989 if (AddcSubcNode == HiAddSub->getNode() ||
13005 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->
useMulOps() &&
13019 return SDValue(AddeSubeNode, 0);
13030 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13040 return SDValue(AddeSubeNode, 0);
13052 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13061 SDNode *UmlalNode =
nullptr;
13086 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13100 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13105 SDNode* AddcNode =
N->getOperand(2).getNode();
13106 SDNode* AddeNode =
N->getOperand(3).getNode();
13114 {N->getOperand(0), N->getOperand(1),
13115 AddcNode->getOperand(0), AddcNode->getOperand(1)});
13139 int32_t imm =
C->getSExtValue();
13140 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13145 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
N->getOperand(0),
RHS);
13160 int64_t imm =
C->getSExtValue();
13171 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
13172 N->getOperand(0),
RHS,
N->getOperand(2));
13184 if (!Subtarget->hasMVEIntegerOps())
13197 SetCC =
N->getOperand(0);
13200 CC = cast<CondCodeSDNode>(SetCC->
getOperand(2))->get();
13201 TrueVal =
N->getOperand(1);
13202 FalseVal =
N->getOperand(2);
13204 LHS =
N->getOperand(0);
13205 RHS =
N->getOperand(1);
13206 CC = cast<CondCodeSDNode>(
N->getOperand(4))->get();
13207 TrueVal =
N->getOperand(2);
13208 FalseVal =
N->getOperand(3);
13213 unsigned int Opcode = 0;
13242 switch (TrueVal->getOpcode()) {
13261 if (TrueVal !=
LHS || FalseVal !=
RHS)
13264 EVT LeftType =
LHS->getValueType(0);
13265 EVT RightType =
RHS->getValueType(0);
13268 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13272 if (VectorScalarType != MVT::i32)
13280 if (VectorScalarType != MVT::i32)
13293 EVT VT =
N->getValueType(0);
13301 Shft =
N->getOperand(0);
13307 cast<CondCodeSDNode>(Cmp.getOperand(2))->get() !=
ISD::SETLT ||
13308 Cmp.getOperand(0) !=
N->getOperand(1) ||
13309 Cmp.getOperand(1) !=
N->getOperand(2))
13311 Shft =
N->getOperand(1);
13323 ScalarType = MVT::i8;
13326 case (1 << 15) - 1:
13327 ScalarType = MVT::i16;
13330 case (1ULL << 31) - 1:
13331 ScalarType = MVT::i32;
13362 unsigned LegalLanes = 128 / (ShftAmt + 1);
13386 for (
unsigned I = 0;
I < NumParts; ++
I) {
13403 if (!Subtarget->hasMVEIntegerOps())
13418 if (
N->getOperand(0).getOpcode() !=
ISD::XOR)
13428 if (!Const || !Const->isOne())
13446 EVT VT =
N->getValueType(0);
13448 if (!Subtarget->hasMVEIntegerOps() ||
13477 Opc = Intrinsic::arm_mve_vctp64;
13480 Opc = Intrinsic::arm_mve_vctp32;
13483 Opc = Intrinsic::arm_mve_vctp16;
13486 Opc = Intrinsic::arm_mve_vctp8;
13552 EVT VT =
N->getValueType(0);
13558 switch (
Op.getOpcode()) {
13575 !isa<ConstantSDNode>(N0) && N1->
hasOneUse()) {
13583 unsigned N0RedOp = 0;
13590 unsigned N1RedOp = 0;
13604 if (
SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13606 if (
SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13613 auto DistrubuteVecReduceLoad = [&](
SDValue N0,
SDValue N1,
bool IsForward) {
13627 LoadSDNode *Load0 = dyn_cast<LoadSDNode>(N0);
13628 LoadSDNode *Load1 = dyn_cast<LoadSDNode>(N1);
13637 if (!BaseLocDecomp0.getBase() ||
13638 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13639 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13641 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13643 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13653 if (IsBefore < 0) {
13656 }
else if (IsBefore > 0) {
13669 }
else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13679 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13689 if (
SDValue R = DistrubuteVecReduceLoad(N0, N1,
true))
13691 if (
SDValue R = DistrubuteVecReduceLoad(N1, N0,
false))
13698 if (!Subtarget->hasMVEIntegerOps())
13704 EVT VT =
N->getValueType(0);
13709 if (VT != MVT::i64)
13720 auto MakeVecReduce = [&](
unsigned Opcode,
unsigned OpcodeA,
SDValue NA,
13738 std::tie(Ops[0], Ops[1]) = DAG.
SplitScalar(NA, dl, MVT::i32, MVT::i32);
13740 unsigned S = VecRed->
getOpcode() == OpcodeA ? 2 : 0;
13789 "Expected shift op");
13807 if (
auto *Const = dyn_cast<ConstantSDNode>(N1->
getOperand(1))) {
13808 if (Const->getAPIntValue().ult(256))
13811 Const->getAPIntValue().sgt(-256))
13827 (
N->getOperand(0).getOpcode() ==
ISD::SHL ||
13828 N->getOperand(0).getOpcode() ==
ISD::SRL) &&
13829 "Expected XOR(SHIFT) pattern");
13832 auto *XorC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13833 auto *ShiftC = dyn_cast<ConstantSDNode>(
N->getOperand(0).getOperand(1));
13834 if (XorC && ShiftC) {
13835 unsigned MaskIdx, MaskLen;
13836 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13837 unsigned ShiftAmt = ShiftC->getZExtValue();
13838 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
13839 if (
N->getOperand(0).getOpcode() ==
ISD::SHL)
13840 return MaskIdx == ShiftAmt && MaskLen == (
BitWidth - ShiftAmt);
13841 return MaskIdx == 0 && MaskLen == (
BitWidth - ShiftAmt);
13851 N->getOperand(0).getOpcode() ==
ISD::SRL) ||
13853 N->getOperand(0).getOpcode() ==
ISD::SHL)) &&
13854 "Expected shift-shift mask");
13867 return Subtarget->hasMVEIntegerOps() &&
isTypeLegal(VT);
13871 if (!Subtarget->hasNEON()) {
13890 return Subtarget->hasFP64();
13893 return Subtarget->hasMVEFloatOps();
13922 if (ST->isThumb() && ST->isThumb1Only())
13926 for (
auto *U :
N->uses()) {
13927 switch(U->getOpcode()) {
13940 if (isa<ConstantSDNode>(U->getOperand(0)) ||
13941 isa<ConstantSDNode>(U->getOperand(1)))
13945 if (U->getOperand(0).getOpcode() ==
ISD::SHL ||
13946 U->getOperand(1).getOpcode() ==
ISD::SHL)
13956 if (
N->getOperand(0).getOpcode() !=
ISD::SHL)
13961 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13962 auto *C2 = dyn_cast<ConstantSDNode>(SHL.getOperand(1));
13963 if (!C1ShlC2 || !C2)
13966 APInt C2Int = C2->getAPIntValue();
13967 APInt C1Int = C1ShlC2->getAPIntValue();
13969 if (C2Int.
uge(C2Width))
13975 if ((C1Int & Mask) != C1Int)
13982 auto LargeImm = [](
const APInt &Imm) {
13983 unsigned Zeros = Imm.countl_zero() + Imm.countr_zero();
13984 return Imm.getBitWidth() - Zeros > 8;
13987 if (LargeImm(C1Int) || LargeImm(C2Int))
13999 SHL.dump();
N->dump());
14034 if (CSINC.getOpcode() !=
ARMISD::CSINC || !CSINC.hasOneUse())
14043 CSINC.getOperand(0)),
14044 CSINC.getOperand(1), CSINC.getOperand(2),
14045 CSINC.getOperand(3));
14064 if (!Subtarget->hasMVEIntegerOps() || !
N->getValueType(0).isVector())
14106 if (!Subtarget->hasVMLxForwarding())
14125 EVT VT =
N->getValueType(0);
14136 EVT VT =
N->getValueType(0);
14137 if (VT != MVT::v2i64)
14146 EVT VT = cast<VTSDNode>(
Op->getOperand(1))->getVT();
14148 return Op->getOperand(0);
14162 And =
And->getOperand(0);
14167 Mask = Mask->getOperand(0);
14170 Mask.getValueType() != MVT::v4i32)
14176 return And->getOperand(0);
14181 if (
SDValue Op0 = IsSignExt(N0)) {
14182 if (
SDValue Op1 = IsSignExt(N1)) {
14188 if (
SDValue Op0 = IsZeroExt(N0)) {
14189 if (
SDValue Op1 = IsZeroExt(N1)) {
14204 EVT VT =
N->getValueType(0);
14205 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14216 if (VT != MVT::i32)
14223 int64_t MulAmt =
C->getSExtValue();
14224 unsigned ShiftAmt = llvm::countr_zero<uint64_t>(MulAmt);
14226 ShiftAmt = ShiftAmt & (32 - 1);
14231 MulAmt >>= ShiftAmt;
14234 if (llvm::has_single_bit<uint32_t>(MulAmt - 1)) {
14242 }
else if (llvm::has_single_bit<uint32_t>(MulAmt + 1)) {
14254 if (llvm::has_single_bit<uint32_t>(MulAmtAbs + 1)) {
14262 }
else if (llvm::has_single_bit<uint32_t>(MulAmtAbs - 1)) {
14292 if (
N->getValueType(0) != MVT::i32)
14301 if (C1 == 255 || C1 == 65535)
14304 SDNode *N0 =
N->getOperand(0).getNode();
14318 if (!C2 || C2 >= 32)
14362 if (Trailing == C2 && C2 + C3 < 32) {
14375 if (Leading == C2 && C2 + C3 < 32) {
14395 EVT VT =
N->getValueType(0);
14399 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14402 APInt SplatBits, SplatUndef;
14403 unsigned SplatBitSize;
14405 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14406 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14407 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14408 SplatBitSize == 64) {
14442 if (!Subtarget->hasV6Ops() ||
14443 (Subtarget->isThumb() &&
14444 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14447 SDValue SRL = OR->getOperand(0);
14448 SDValue SHL = OR->getOperand(1);
14451 SRL = OR->getOperand(1);
14452 SHL = OR->getOperand(0);
14459 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) ||
14463 SDNode *SMULLOHI = SRL.getOperand(0).getNode();
14464 if (SRL.getOperand(0) !=
SDValue(SMULLOHI, 0) ||
14465 SHL.getOperand(0) !=
SDValue(SMULLOHI, 1))
14484 unsigned Opcode = 0;
14485 if (
isS16(OpS16, DAG))
14503 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6T2Ops())
14506 EVT VT =
N->getValueType(0);
14521 if (VT != MVT::i32)
14534 if (Mask == 0xffff)
14541 if ((Val & ~Mask) != Val)
14566 (Mask == ~Mask2)) {
14569 if (Subtarget->hasDSP() &&
14570 (Mask == 0xffff || Mask == 0xffff0000))
14583 (~Mask == Mask2)) {
14586 if (Subtarget->hasDSP() &&
14587 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14660 EVT VT =
N->getValueType(0);
14665 auto IsFreelyInvertable = [&](
SDValue V) {
14672 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14688 EVT VT =
N->getValueType(0);
14694 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14695 VT == MVT::v8i1 || VT == MVT::v16i1))
14698 APInt SplatBits, SplatUndef;
14699 unsigned SplatBitSize;
14701 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14702 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14703 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14704 SplatBitSize == 64) {
14740 unsigned SplatBitSize;
14743 APInt SplatBits0, SplatBits1;
14747 if (BVN0 && BVN0->
isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
14748 HasAnyUndefs) && !HasAnyUndefs) {
14749 if (BVN1 && BVN1->
isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
14750 HasAnyUndefs) && !HasAnyUndefs) {
14755 SplatBits0 == ~SplatBits1) {
14785 EVT VT =
N->getValueType(0);
14800 if (Subtarget->hasMVEIntegerOps()) {
14831 ToMask = ~N->getConstantOperandAPInt(2);
14837 isa<ConstantSDNode>(
From->getOperand(1))) {
14838 APInt Shift =
From->getConstantOperandAPInt(1);
14851 unsigned LastActiveBitInA =
A.countr_zero();
14852 unsigned FirstActiveBitInB =
B.getBitWidth() -
B.countl_zero() - 1;
14853 return LastActiveBitInA - 1 == FirstActiveBitInB;
14858 APInt ToMask, FromMask;
14866 APInt NewToMask, NewFromMask;
14868 if (NewFrom !=
From)
14872 if ((NewToMask & ToMask).getBoolValue())
14897 unsigned InvMask =
N->getConstantOperandVal(2);
14899 unsigned Width = llvm::bit_width<unsigned>(~InvMask) - LSB;
14901 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14902 "undefined behavior");
14903 unsigned Mask = (1u << Width) - 1;
14905 if ((Mask & (~Mask2)) == 0)
14907 N->getOperand(0), N1.
getOperand(0),
N->getOperand(2));
14914 APInt ToMask1, FromMask1;
14917 APInt ToMask2, FromMask2;
14923 APInt NewFromMask = FromMask1 | FromMask2;
14924 APInt NewToMask = ToMask1 | ToMask2;
14926 EVT VT =
N->getValueType(0);
14929 if (NewFromMask[0] == 0)
14941 APInt ToMask1 = ~N->getConstantOperandAPInt(2);
14942 APInt ToMask2 = ~N0.getConstantOperandAPInt(2);
14944 if (!N0.
hasOneUse() || (ToMask1 & ToMask2) != 0 ||
14948 EVT VT =
N->getValueType(0);
14951 N->getOperand(1),
N->getOperand(2));
14965 SDValue CSInc = Cmp->getOperand(0);
15015 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ)
15016 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15019 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
15021 N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15034 SDValue InDouble =
N->getOperand(0);
15043 !cast<LoadSDNode>(InNode)->isVolatile()) {
15049 SDValue BasePtr = LD->getBasePtr();
15051 DAG.
getLoad(MVT::i32,
DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
15052 LD->getAlign(), LD->getMemOperand()->getFlags());
15058 LD->getPointerInfo().getWithOffset(4),
15060 LD->getMemOperand()->getFlags());
15072 isa<ConstantSDNode>(InDouble.
getOperand(1))) {
15093 if (!Subtarget->
isLittle() && BVSwap)
15103 if (isa<ConstantSDNode>(BV.
getOperand(2))) {
15111 if (!Subtarget->
isLittle() && BVSwap)
15156 if (Copy.getValueType() == MVT::f32 &&
15158 bool HasGlue = Copy->getNumOperands() == 3;
15159 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1),
15160 HasGlue ? Copy->getOperand(2) :
SDValue()};
15161 EVT OutTys[] = {
N->getValueType(0), MVT::Other, MVT::Glue};
15179 if (
LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Op0)) {
15180 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15181 LN0->getMemoryVT() == MVT::i16) {
15184 LN0->getBasePtr(), LN0->getMemOperand());
15202 EVT VT =
N->getValueType(0);
15236 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
15237 for (
unsigned i = 0; i < NumElts; ++i) {
15238 SDNode *Elt =
N->getOperand(i).getNode();
15255 if (
N->getNumOperands() == 2)
15261 EVT VT =
N->getValueType(0);
15267 for (
unsigned i = 0; i < NumElts; ++i) {
15293 EVT VT =
N->getValueType(0);
15301 assert(EltVT == MVT::f32 &&
"Unexpected type!");
15306 Use->getValueType(0).isFloatingPoint())
15314 unsigned NumOfBitCastedElts = 0;
15316 unsigned NumOfRelevantElts = NumElts;
15317 for (
unsigned Idx = 0;
Idx < NumElts; ++
Idx) {
15322 ++NumOfBitCastedElts;
15323 }
else if (Elt.
isUndef() || isa<ConstantSDNode>(Elt))
15326 --NumOfRelevantElts;
15330 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15348 for (
unsigned Idx = 0 ;
Idx < NumElts; ++
Idx) {
15353 V->getOperand(0).getValueType() == MVT::i32)
15355 V = V.getOperand(0);
15372 EVT VT =
N->getValueType(0);
15379 if (
Op->getOperand(0).getValueType() == VT)
15380 return Op->getOperand(0);
15395 if (
Op.getValueType() == MVT::i32) {
15406 EVT VT =
N->getValueType(0);
15411 if (ST->isLittle())
15421 if (
Op->getOperand(0).getValueType() == VT)
15422 return Op->getOperand(0);
15431 if (!Subtarget->hasMVEIntegerOps())
15434 EVT VT =
N->getValueType(0);
15465 EVT VT =
N->getValueType(0);
15466 SDNode *Elt =
N->getOperand(1).getNode();
15481 Vec, V,
N->getOperand(2));
15491 EVT VT =
N->getValueType(0);
15500 Ext.getOperand(0).getValueType() == MVT::f32)
15501 Ext = Ext.getOperand(0);
15503 !isa<ConstantSDNode>(Ext.getOperand(1)) ||
15504 Ext.getConstantOperandVal(1) % 2 != 0)
15506 if (Ext->use_size() == 1 &&
15511 SDValue Op0 = Ext.getOperand(0);
15514 unsigned Lane = Ext.getConstantOperandVal(1);
15520 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15521 isa<ConstantSDNode>(V->getOperand(1)) &&
15522 V->getConstantOperandVal(1) == Lane + 1 &&
15523 V->getOperand(0).getResNo() == ResNo;
15525 if (OtherIt == Op0->
uses().end())
15530 SDValue OtherExt(*OtherIt, 0);
15543 DCI.
DAG.
getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32));
15555 EVT VT =
N->getValueType(0);
15561 if (VT == MVT::f16 &&
X.getValueType() == MVT::i32)
15563 if (VT == MVT::i32 &&
X.getValueType() == MVT::f16)
15565 if (VT == MVT::f32 &&
X.getValueType() == MVT::i32)
15568 while (
X.getValueType() != VT &&
X->getOpcode() ==
ISD::BITCAST)
15569 X =
X->getOperand(0);
15570 if (
X.getValueType() == VT)
15576 isa<ConstantSDNode>(
N->getOperand(1)) &&
15578 return Op0.
getOperand(
N->getConstantOperandVal(1));
15583 isa<ConstantSDNode>(
N->getOperand(1)) &&
15588 unsigned Offset =
N->getConstantOperandVal(1);
15600 unsigned Idx =
N->getConstantOperandVal(1);
15614 EVT VT =
N->getValueType(0);
15618 cast<VTSDNode>(
N->getOperand(1))->getVT() ==
15619 Op.getOperand(0).getValueType().getScalarType())
15629 SDValue SubVec =
N->getOperand(1);
15630 uint64_t IdxVal =
N->getConstantOperandVal(2);
15641 if (IdxVal == 0 && Vec.
isUndef())
15647 (IdxVal != 0 && IdxVal != NumSubElts))
15720 EVT VT =
N->getValueType(0);
15731 unsigned HalfElts = NumElts/2;
15733 for (
unsigned n = 0; n < NumElts; ++n) {
15736 if (MaskElt < (
int)HalfElts)
15738 else if (MaskElt >= (
int)NumElts && MaskElt < (
int)(NumElts + HalfElts))
15739 NewElt = HalfElts + MaskElt - NumElts;
15766 bool SimpleConstIncOnly,
15774 bool isLoadOp =
true;
15775 bool isLaneOp =
false;
15778 bool hasAlignment =
true;
15779 unsigned NewOpc = 0;
15780 unsigned NumVecs = 0;
15781 if (
Target.isIntrinsic) {
15782 unsigned IntNo =
N->getConstantOperandVal(1);
15786 case Intrinsic::arm_neon_vld1:
15790 case Intrinsic::arm_neon_vld2:
15794 case Intrinsic::arm_neon_vld3:
15798 case Intrinsic::arm_neon_vld4:
15802 case Intrinsic::arm_neon_vld1x2:
15805 hasAlignment =
false;
15807 case Intrinsic::arm_neon_vld1x3:
15810 hasAlignment =
false;
15812 case Intrinsic::arm_neon_vld1x4:
15815 hasAlignment =
false;
15817 case Intrinsic::arm_neon_vld2dup:
15821 case Intrinsic::arm_neon_vld3dup:
15825 case Intrinsic::arm_neon_vld4dup:
15829 case Intrinsic::arm_neon_vld2lane:
15834 case Intrinsic::arm_neon_vld3lane:
15839 case Intrinsic::arm_neon_vld4lane:
15844 case Intrinsic::arm_neon_vst1:
15849 case Intrinsic::arm_neon_vst2:
15854 case Intrinsic::arm_neon_vst3:
15859 case Intrinsic::arm_neon_vst4:
15864 case Intrinsic::arm_neon_vst2lane:
15870 case Intrinsic::arm_neon_vst3lane:
15876 case Intrinsic::arm_neon_vst4lane:
15882 case Intrinsic::arm_neon_vst1x2:
15886 hasAlignment =
false;
15888 case Intrinsic::arm_neon_vst1x3:
15892 hasAlignment =
false;
15894 case Intrinsic::arm_neon_vst1x4:
15898 hasAlignment =
false;
15903 switch (
N->getOpcode()) {
15939 VecTy =
N->getValueType(0);
15940 }
else if (
Target.isIntrinsic) {
15941 VecTy =
N->getOperand(
Target.AddrOpIdx + 1).getValueType();
15944 "Node has to be a load, a store, or an intrinsic!");
15945 VecTy =
N->getOperand(1).getValueType();
15953 if (isLaneOp || isVLDDUPOp)
15956 if (NumBytes >= 3 * 16 &&
User.ConstInc != NumBytes) {
15962 if (SimpleConstIncOnly &&
User.ConstInc != NumBytes)
15968 EVT AlignedVecTy = VecTy;
15985 if (isa<LSBaseSDNode>(
N)) {
15988 assert(NumVecs == 1 &&
"Unexpected multi-element generic load/store.");
15989 assert(!isLaneOp &&
"Unexpected generic load/store lane.");
16000 Alignment =
Align(1);
16006 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16008 for (n = 0; n < NumResultVecs; ++n)
16009 Tys[n] = AlignedVecTy;
16010 Tys[n++] = MVT::i32;
16011 Tys[n] = MVT::Other;
16026 unsigned LastOperand =
16027 hasAlignment ?
N->getNumOperands() - 1 :
N->getNumOperands();
16028 for (
unsigned i =
Target.AddrOpIdx + 1; i < LastOperand; ++i)
16037 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::STORE) {
16048 for (
unsigned i = 0; i < NumResultVecs; ++i)
16053 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::LOAD) {
16054 SDValue &LdVal = NewResults[0];
16090 switch (
N->getOpcode()) {
16093 if (isa<ConstantSDNode>(
N->getOperand(1))) {
16094 *
Ptr =
N->getOperand(0);
16095 *CInc =
N->getOperand(1);
16101 if (isa<ConstantSDNode>(
N->getOperand(2))) {
16102 *
Ptr =
N->getOperand(1);
16103 *CInc =
N->getOperand(2);
16138 const unsigned AddrOpIdx = ((isIntrinsic ||
isStore) ? 2 : 1);
16147 UE =
Addr.getNode()->use_end(); UI != UE; ++UI) {
16149 if (UI.getUse().getResNo() !=
Addr.getResNo() ||
16154 unsigned ConstInc =
16172 if (UI.getUse().getResNo() !=
Base.getResNo() ||
User ==
Addr.getNode() ||
16177 unsigned UserOffset =
16180 if (!UserOffset || UserOffset <=
Offset)
16183 unsigned NewConstInc = UserOffset -
Offset;
16193 unsigned NumValidUpd = BaseUpdates.
size();
16194 for (
unsigned I = 0;
I < NumValidUpd;) {
16198 std::swap(BaseUpdates[
I], BaseUpdates[NumValidUpd]);
16206 BaseUpdates.
resize(NumValidUpd);
16211 std::stable_sort(BaseUpdates.
begin(), BaseUpdates.
end(),
16213 return LHS.ConstInc < RHS.ConstInc;
16242 unsigned IntNo =
N->getConstantOperandVal(1);
16243 if (IntNo == Intrinsic::arm_mve_vst2q &&
N->getConstantOperandVal(5) != 1)
16245 if (IntNo == Intrinsic::arm_mve_vst4q &&
N->getConstantOperandVal(7) != 3)
16250 UE =
Addr.getNode()->use_end();
16254 UI.getUse().getResNo() !=
Addr.getResNo())
16270 bool isLoadOp =
true;
16271 unsigned NewOpc = 0;
16272 unsigned NumVecs = 0;
16276 case Intrinsic::arm_mve_vld2q:
16280 case Intrinsic::arm_mve_vld4q:
16284 case Intrinsic::arm_mve_vst2q:
16289 case Intrinsic::arm_mve_vst4q:
16299 VecTy =
N->getValueType(0);
16301 VecTy =
N->getOperand(3).getValueType();
16315 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16317 for (n = 0; n < NumResultVecs; ++n)
16319 Tys[n++] = MVT::i32;
16320 Tys[n] = MVT::Other;
16329 for (
unsigned i = 3; i <
N->getNumOperands(); ++i)
16337 for (
unsigned i = 0; i < NumResultVecs; ++i)
16356 EVT VT =
N->getValueType(0);
16362 SDNode *VLD =
N->getOperand(0).getNode();
16365 unsigned NumVecs = 0;
16366 unsigned NewOpc = 0;
16368 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16371 }
else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16374 }
else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16387 if (UI.getUse().getResNo() == NumVecs)
16391 VLDLaneNo !=
User->getConstantOperandVal(1))
16398 for (n = 0; n < NumVecs; ++n)
16400 Tys[n] = MVT::Other;
16411 unsigned ResNo = UI.getUse().
getResNo();
16413 if (ResNo == NumVecs)
16421 std::vector<SDValue> VLDDupResults;
16422 for (
unsigned n = 0; n < NumVecs; ++n)
16436 EVT VT =
N->getValueType(0);
16439 if (Subtarget->hasMVEIntegerOps()) {
16443 ExtractVT = MVT::i32;
16445 N->getOperand(0),
N->getOperand(1));
16457 Op =
Op.getOperand(0);
16462 unsigned EltSize =
Op.getScalarValueSizeInBits();
16464 unsigned Imm =
Op.getConstantOperandVal(0);
16480 if (Subtarget->hasMVEIntegerOps()) {
16483 if (
Op.getValueType() == MVT::f32)
16486 else if (
Op.getValueType() == MVT::f16)
16491 if (!Subtarget->hasNEON())
16498 if (LD &&
Op.hasOneUse() && LD->isUnindexed() &&
16499 LD->getMemoryVT() ==
N->getValueType(0).getVectorElementType()) {
16500 SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1),
16505 LD->getMemoryVT(), LD->getMemOperand());
16516 EVT VT =
N->getValueType(0);
16538 assert(StVT != VT &&
"Cannot truncate to the same type");
16548 if (0 != (NumElems * FromEltSz) % ToEltSz)
16551 unsigned SizeRatio = FromEltSz / ToEltSz;
16556 NumElems * SizeRatio);
16562 for (
unsigned i = 0; i < NumElems; ++i)
16576 MVT StoreType = MVT::i8;
16578 if (TLI.
isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16597 unsigned E = (ToEltSz * NumElems) / StoreType.
getSizeInBits();
16598 for (
unsigned I = 0;
I < E;
I++) {
16629 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16632 unsigned NumElements = 4;
16649 unsigned Off0 = Rev ? NumElts : 0;
16650 unsigned Off1 = Rev ? 0 : NumElts;
16652 for (
unsigned I = 0;
I < NumElts;
I += 2) {
16653 if (M[
I] >= 0 && M[
I] != (
int)(Off0 +
I / 2))
16655 if (M[
I + 1] >= 0 && M[
I + 1] != (
int)(Off1 +
I / 2))
16662 if (
auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Trunc.
getOperand(0)))
16663 if (isVMOVNShuffle(Shuffle,
false) || isVMOVNShuffle(Shuffle,
true))
16683 unsigned NewOffset = i * NumElements * ToEltVT.
getSizeInBits() / 8;
16698 NewToVT, Alignment, MMOFlags, AAInfo);
16731 unsigned NewOffset =
16739 NewToVT, Alignment, MMOFlags, AAInfo);
16761 {Extract.getOperand(0), Extract.getOperand(1)});
16792 if (Subtarget->hasNEON())
16796 if (Subtarget->hasMVEFloatOps())
16800 if (Subtarget->hasMVEIntegerOps()) {
16876 if (!Subtarget->hasNEON())
16880 if (!
Op.getValueType().isVector() || !
Op.getValueType().isSimple() ||
16885 if (!isa<BuildVectorSDNode>(ConstVec))
16888 MVT FloatTy =
Op.getSimpleValueType().getVectorElementType();
16890 MVT IntTy =
N->getSimpleValueType(0).getVectorElementType();
16892 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
16893 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
16904 if (
C == -1 ||
C == 0 ||
C > 32)
16909 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
16910 Intrinsic::arm_neon_vcvtfp2fxu;
16913 DAG.
getConstant(IntrinsicOpcode, dl, MVT::i32),
Op->getOperand(0),
16916 if (IntBits < FloatBits)
16924 if (!Subtarget->hasMVEFloatOps())
16932 EVT VT =
N->getValueType(0);
16937 auto isIdentitySplat = [&](
SDValue Op,
bool NSZ) {
16941 uint64_t ImmVal =
Op.getOperand(0).getConstantOperandVal(0);
16942 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
16944 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
16957 if (!isIdentitySplat(Op1.
getOperand(2), NSZ))
16968 EVT VT =
N->getValueType(0);
16971 if (!
N->getFlags().hasAllowReassociation())
16978 unsigned Opc =
A.getConstantOperandVal(0);
16979 if (Opc != Intrinsic::arm_mve_vcmlaq)
16984 A.getOperand(3),
A.getOperand(4));
17016 if (!Subtarget->hasNEON())
17020 unsigned OpOpcode =
Op.getNode()->getOpcode();
17021 if (!
N->getValueType(0).isVector() || !
N->getValueType(0).isSimple() ||
17025 SDValue ConstVec =
N->getOperand(1);
17026 if (!isa<BuildVectorSDNode>(ConstVec))
17029 MVT FloatTy =
N->getSimpleValueType(0).getVectorElementType();
17031 MVT IntTy =
Op.getOperand(0).getSimpleValueType().getVectorElementType();
17033 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
17034 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17045 if (
C == -1 ||
C == 0 ||
C > 32)
17051 if (IntBits < FloatBits)
17053 dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
17056 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfxs2fp :
17057 Intrinsic::arm_neon_vcvtfxu2fp;
17066 if (!ST->hasMVEIntegerOps())
17070 EVT ResVT =
N->getValueType(0);
17098 EVT AVT =
A.getValueType();
17104 auto ExtendIfNeeded = [&](
SDValue A,
unsigned ExtendCode) {
17105 EVT AVT =
A.getValueType();
17117 if (ExtTypeMatches(
A, ExtTypes))
17118 return ExtendIfNeeded(
A, ExtendCode);
17121 auto IsPredVADDV = [&](
MVT RetTy,
unsigned ExtendCode,
17128 if (Ext->getOpcode() != ExtendCode)
17131 if (ExtTypeMatches(
A, ExtTypes))
17132 return ExtendIfNeeded(
A, ExtendCode);
17145 if (ResVT !=
RetTy)
17148 if (
Mul->getOpcode() == ExtendCode &&
17149 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17151 Mul =
Mul->getOperand(0);
17160 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17161 A = ExtendIfNeeded(
A, ExtendCode);
17162 B = ExtendIfNeeded(
B, ExtendCode);
17180 if (
Mul->getOpcode() == ExtendCode &&
17181 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17183 Mul =
Mul->getOperand(0);
17192 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17193 A = ExtendIfNeeded(
A, ExtendCode);
17194 B = ExtendIfNeeded(
B, ExtendCode);
17205 EVT VT = Ops[0].getValueType();
17206 if (VT == MVT::v16i8) {
17208 "Unexpected illegal long reduction opcode");
17213 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
17216 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
17237 if (IsVMLAV(MVT::i64,
ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17240 if (IsVMLAV(MVT::i64,
ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17304 Op =
Op->getOperand(1);
17306 Op->getOperand(0)->getOpcode() ==
ISD::MUL) {
17308 if (
Mul->getOperand(0) ==
Mul->getOperand(1) &&
17325 unsigned VecOp =
N->getOperand(0).getValueType().isVector() ? 0 : 2;
17326 auto *Shuf = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp));
17327 if (!Shuf || !Shuf->getOperand(1).isUndef())
17332 APInt SetElts(Mask.size(), 0);
17333 for (
int E : Mask) {
17334 if (E < 0 || E >= (
int)Mask.size())
17341 if (
N->getNumOperands() != VecOp + 1) {
17342 auto *Shuf2 = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp + 1));
17343 if (!Shuf2 || !Shuf2->getOperand(1).isUndef() || Shuf2->getMask() != Mask)
17349 if (
Op.getValueType().isVector())
17361 unsigned IsTop =
N->getConstantOperandVal(2);
17368 if (Op0->
isUndef() && !IsTop)
17382 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17384 APInt Op0DemandedElts =
17385 IsTop ? Op1DemandedElts
17400 unsigned IsTop =
N->getConstantOperandVal(2);
17402 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17403 APInt Op0DemandedElts =
17415 EVT VT =
N->getValueType(0);
17419 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(
LHS);
17420 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(
RHS);
17422 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
17423 LHS.getOperand(1).isUndef() &&
RHS.getOperand(1).isUndef() &&
17427 LHS.getOperand(0),
RHS.getOperand(0));
17441 if (
auto C = dyn_cast<ConstantSDNode>(
N->getOperand(2))) {
17442 int ShiftAmt =
C->getSExtValue();
17443 if (ShiftAmt == 0) {
17449 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17450 unsigned NewOpcode =
17466 unsigned IntNo =
N->getConstantOperandVal(0);
17477 case Intrinsic::arm_neon_vshifts:
17478 case Intrinsic::arm_neon_vshiftu:
17479 case Intrinsic::arm_neon_vrshifts:
17480 case Intrinsic::arm_neon_vrshiftu:
17481 case Intrinsic::arm_neon_vrshiftn:
17482 case Intrinsic::arm_neon_vqshifts:
17483 case Intrinsic::arm_neon_vqshiftu:
17484 case Intrinsic::arm_neon_vqshiftsu:
17485 case Intrinsic::arm_neon_vqshiftns:
17486 case Intrinsic::arm_neon_vqshiftnu:
17487 case Intrinsic::arm_neon_vqshiftnsu:
17488 case Intrinsic::arm_neon_vqrshiftns:
17489 case Intrinsic::arm_neon_vqrshiftnu:
17490 case Intrinsic::arm_neon_vqrshiftnsu: {
17491 EVT VT =
N->getOperand(1).getValueType();
17493 unsigned VShiftOpc = 0;
17496 case Intrinsic::arm_neon_vshifts:
17497 case Intrinsic::arm_neon_vshiftu:
17502 if (
isVShiftRImm(
N->getOperand(2), VT,
false,
true, Cnt)) {
17509 case Intrinsic::arm_neon_vrshifts:
17510 case Intrinsic::arm_neon_vrshiftu:
17515 case Intrinsic::arm_neon_vqshifts:
17516 case Intrinsic::arm_neon_vqshiftu:
17521 case Intrinsic::arm_neon_vqshiftsu:
17526 case Intrinsic::arm_neon_vrshiftn:
17527 case Intrinsic::arm_neon_vqshiftns:
17528 case Intrinsic::arm_neon_vqshiftnu:
17529 case Intrinsic::arm_neon_vqshiftnsu:
17530 case Intrinsic::arm_neon_vqrshiftns:
17531 case Intrinsic::arm_neon_vqrshiftnu:
17532 case Intrinsic::arm_neon_vqrshiftnsu:
17544 case Intrinsic::arm_neon_vshifts:
17545 case Intrinsic::arm_neon_vshiftu:
17548 case Intrinsic::arm_neon_vrshifts:
17551 case Intrinsic::arm_neon_vrshiftu:
17554 case Intrinsic::arm_neon_vrshiftn:
17557 case Intrinsic::arm_neon_vqshifts:
17560 case Intrinsic::arm_neon_vqshiftu:
17563 case Intrinsic::arm_neon_vqshiftsu:
17566 case Intrinsic::arm_neon_vqshiftns:
17569 case Intrinsic::arm_neon_vqshiftnu:
17572 case Intrinsic::arm_neon_vqshiftnsu:
17575 case Intrinsic::arm_neon_vqrshiftns:
17578 case Intrinsic::arm_neon_vqrshiftnu:
17581 case Intrinsic::arm_neon_vqrshiftnsu:
17587 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17588 N->getOperand(1), DAG.
getConstant(Cnt, dl, MVT::i32));
17591 case Intrinsic::arm_neon_vshiftins: {
17592 EVT VT =
N->getOperand(1).getValueType();
17594 unsigned VShiftOpc = 0;
17598 else if (
isVShiftRImm(
N->getOperand(3), VT,
false,
true, Cnt))
17605 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17606 N->getOperand(1),
N->getOperand(2),
17610 case Intrinsic::arm_neon_vqrshifts:
17611 case Intrinsic::arm_neon_vqrshiftu:
17615 case Intrinsic::arm_mve_vqdmlah:
17616 case Intrinsic::arm_mve_vqdmlash:
17617 case Intrinsic::arm_mve_vqrdmlah:
17618 case Intrinsic::arm_mve_vqrdmlash:
17619 case Intrinsic::arm_mve_vmla_n_predicated:
17620 case Intrinsic::arm_mve_vmlas_n_predicated:
17621 case Intrinsic::arm_mve_vqdmlah_predicated:
17622 case Intrinsic::arm_mve_vqdmlash_predicated:
17623 case Intrinsic::arm_mve_vqrdmlah_predicated:
17624 case Intrinsic::arm_mve_vqrdmlash_predicated: {
17629 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
17636 case Intrinsic::arm_mve_minv:
17637 case Intrinsic::arm_mve_maxv:
17638 case Intrinsic::arm_mve_minav:
17639 case Intrinsic::arm_mve_maxav:
17640 case Intrinsic::arm_mve_minv_predicated:
17641 case Intrinsic::arm_mve_maxv_predicated:
17642 case Intrinsic::arm_mve_minav_predicated:
17643 case Intrinsic::arm_mve_maxav_predicated: {
17646 unsigned BitWidth =
N->getOperand(2)->getValueType(0).getScalarSizeInBits();
17653 case Intrinsic::arm_mve_addv: {
17656 bool Unsigned =
N->getConstantOperandVal(2);
17661 case Intrinsic::arm_mve_addlv:
17662 case Intrinsic::arm_mve_addlv_predicated: {
17665 bool Unsigned =
N->getConstantOperandVal(2);
17666 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
17671 for (
unsigned i = 1, e =
N->getNumOperands(); i < e; i++)
17694 EVT VT =
N->getValueType(0);
17696 if (ST->isThumb1Only() &&
N->getOpcode() ==
ISD::SHL && VT == MVT::i32 &&
17697 N->getOperand(0)->getOpcode() ==
ISD::AND &&
17698 N->getOperand(0)->hasOneUse()) {
17706 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17715 if (AndMask == 255 || AndMask == 65535)
17719 if (MaskedBits > ShiftAmt) {
17734 if (ST->hasMVEIntegerOps())
17739 switch (
N->getOpcode()) {
17752 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
17753 unsigned VShiftOpc =
17756 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
17772 if (!LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed() ||
17775 EVT FromVT = LD->getValueType(0);
17776 EVT ToVT =
N->getValueType(0);
17783 unsigned NumElements = 0;
17784 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
17786 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
17788 if (NumElements == 0 ||
17798 SDValue BasePtr = LD->getBasePtr();
17799 Align Alignment = LD->getOriginalAlign();
17820 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
17821 Alignment, MMOFlags, AAInfo);
17827 if (FromEltVT == MVT::f16) {
17830 for (
unsigned i = 0; i < Loads.
size(); i++) {
17856 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
17860 EVT VT =
N->getValueType(0);
17864 if (VT == MVT::i32 &&
17865 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
17867 isa<ConstantSDNode>(Lane)) {
17870 switch (
N->getOpcode()) {
17884 if (ST->hasMVEIntegerOps())
17893 if (ST->hasMVEFloatOps())
17904 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
17908 EVT VT =
Op.getValueType();
17911 if (VT != MVT::i32 ||
17913 !isa<ConstantSDNode>(
Op.getOperand(1)) ||
17924 APInt MaxC = Max.getConstantOperandAPInt(1);
17927 !(MinC + 1).isPowerOf2())
17945 EVT VT =
N->getValueType(0);
17948 if (VT == MVT::i32)
17951 if (!ST->hasMVEIntegerOps())
17957 if (VT != MVT::v4i32 && VT != MVT::v8i16)
17960 auto IsSignedSaturate = [&](
SDNode *Min,
SDNode *Max) {
17968 if (VT == MVT::v4i32)
17969 SaturateC =
APInt(32, (1 << 15) - 1,
true);
17971 SaturateC =
APInt(16, (1 << 7) - 1,
true);
17978 MaxC != ~SaturateC)
17983 if (IsSignedSaturate(
N, N0.
getNode())) {
17986 if (VT == MVT::v4i32) {
17987 HalfVT = MVT::v8i16;
17988 ExtVT = MVT::v4i16;
17990 HalfVT = MVT::v16i8;
18005 auto IsUnsignedSaturate = [&](
SDNode *Min) {
18011 if (VT == MVT::v4i32)
18012 SaturateC =
APInt(32, (1 << 16) - 1,
true);
18014 SaturateC =
APInt(16, (1 << 8) - 1,
true);
18023 if (IsUnsignedSaturate(
N)) {
18027 if (VT == MVT::v4i32) {
18028 HalfVT = MVT::v8i16;
18029 ExtConst = 0x0000FFFF;
18031 HalfVT = MVT::v16i8;
18053 const APInt *CV = &
C->getAPIntValue();
18071 SDValue Op0 = CMOV->getOperand(0);
18072 SDValue Op1 = CMOV->getOperand(1);
18073 auto CC = CMOV->getConstantOperandAPInt(2).getLimitedValue();
18074 SDValue CmpZ = CMOV->getOperand(4);
18110 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18117 if ((OrCI & Known.
Zero) != OrCI)
18123 EVT VT =
X.getValueType();
18124 unsigned BitInX = AndC->
logBase2();
18132 for (
unsigned BitInY = 0, NumActiveBits = OrCI.
getActiveBits();
18133 BitInY < NumActiveBits; ++BitInY) {
18134 if (OrCI[BitInY] == 0)
18137 Mask.setBit(BitInY);
18154 switch (
N->getOpcode()) {
18158 if (!isa<ConstantSDNode>(
N.getOperand(1)))
18160 if (!cast<ConstantSDNode>(
N.getOperand(1))->isOne())
18166 auto *Const = dyn_cast<ConstantSDNode>(
N.getOperand(1));
18169 if (Const->isZero())
18171 else if (Const->isOne())
18175 CC = cast<CondCodeSDNode>(
N.getOperand(2))->get();
18179 unsigned IntOp =
N.getConstantOperandVal(1);
18180 if (IntOp != Intrinsic::test_start_loop_iterations &&
18181 IntOp != Intrinsic::loop_decrement_reg)
18207 bool Negate =
false;
18213 Cond =
N->getOperand(1);
18214 Dest =
N->getOperand(2);
18217 CC = cast<CondCodeSDNode>(
N->getOperand(1))->get();
18218 Cond =
N->getOperand(2);
18219 Dest =
N->getOperand(4);
18220 if (
auto *Const = dyn_cast<ConstantSDNode>(
N->getOperand(3))) {
18221 if (!Const->isOne() && !Const->isZero())
18223 Imm = Const->getZExtValue();
18251 assert((IsTrueIfZero(
CC, Imm) || IsFalseIfZero(
CC, Imm)) &&
18252 "unsupported condition");
18257 unsigned IntOp =
Int->getConstantOperandVal(1);
18259 &&
"expected single br user");
18260 SDNode *Br = *
N->use_begin();
18270 if (IntOp == Intrinsic::test_start_loop_iterations) {
18274 if (IsTrueIfZero(
CC, Imm)) {
18275 SDValue Ops[] = {Chain, Setup, Dest};
18280 UpdateUncondBr(Br, Dest, DAG);
18282 SDValue Ops[] = {Chain, Setup, OtherTarget};
18295 DAG.
getVTList(MVT::i32, MVT::Other), Args);
18303 if (
Target == OtherTarget)
18304 UpdateUncondBr(Br, Dest, DAG);
18323 EVT VT =
N->getValueType(0);
18342 LHS->getOperand(0)->getOperand(3),
LHS->getOperand(0)->getOperand(4));
18356 EVT VT =
N->getValueType(0);
18360 SDValue FalseVal =
N->getOperand(0);
18361 SDValue TrueVal =
N->getOperand(1);
18366 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops()) {
18392 N->getOperand(3), Cmp);
18397 N->getOperand(3), NewCmp);
18406 LHS->getOperand(2),
LHS->getOperand(3),
18407 LHS->getOperand(4));
18417 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ ||
18421 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
18423 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
18426 N->getOperand(3),
C);
18433 if (!Subtarget->
isThumb1Only() && Subtarget->hasV5TOps()) {
18469 N->getOperand(3), CPSRGlue.
getValue(1));
18484 N->getOperand(3), CPSRGlue.
getValue(1));
18502 const APInt *TrueConst;
18505 FalseVal.getOperand(0) ==
LHS && FalseVal.getOperand(1) ==
RHS) ||
18509 unsigned ShiftAmount = TrueConst->
logBase2();
18524 if (Known.
Zero == 0xfffffffe)
18527 else if (Known.
Zero == 0xffffff00)
18530 else if (Known.
Zero == 0xffff0000)
18543 EVT DstVT =
N->getValueType(0);
18546 if (ST->hasMVEIntegerOps() && Src.getOpcode() ==
ARMISD::VDUP) {
18547 EVT SrcVT = Src.getValueType();
18555 Src = Src.getOperand(0);
18559 EVT SrcVT = Src.getValueType();
18579 EVT VT =
N->getValueType(0);
18587 if (
N->getNumOperands() == 2 &&
18591 N->getOperand(0).getOperand(1),
18592 N->getOperand(1).getOperand(0),
18593 N->getOperand(1).getOperand(1));
18596 if (
N->getNumOperands() == 2 &&
18599 auto *S0 = cast<ShuffleVectorSDNode>(
N->getOperand(0).getNode());
18600 auto *
S1 = cast<ShuffleVectorSDNode>(
N->getOperand(1).getNode());
18602 if (S0->getOperand(0) ==
S1->getOperand(0) &&
18603 S0->getOperand(1) ==
S1->getOperand(1)) {
18606 Mask.append(
S1->getMask().begin(),
S1->getMask().end());
18626 return Op.getOpcode() == ISD::BUILD_VECTOR ||
18627 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
18628 (Op.getOpcode() == ISD::BITCAST &&
18629 Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR);
18632 for (
unsigned Op = 0;
Op <
N->getNumOperands();
Op++) {
18634 for (
unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
18651 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18652 int NumIns =
N->getNumOperands();
18653 assert((NumIns == 2 || NumIns == 4) &&
18654 "Expected 2 or 4 inputs to an MVETrunc");
18656 if (
N->getNumOperands() == 4)
18660 for (
int I = 0;
I < NumIns;
I++) {
18662 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18682 if (!LD || !LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed())
18685 EVT FromVT = LD->getMemoryVT();
18686 EVT ToVT =
N->getValueType(0);
18693 unsigned NumElements = 0;
18694 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
18696 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
18698 assert(NumElements != 0);
18704 LD->getExtensionType() != NewExtType)
18711 SDValue BasePtr = LD->getBasePtr();
18712 Align Alignment = LD->getOriginalAlign();
18731 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
18732 Alignment, MMOFlags, AAInfo);
18748 EVT VT =
N->getValueType(0);
18750 assert(
N->getNumValues() == 2 &&
"Expected MVEEXT with 2 elements");
18751 assert((VT == MVT::v4i32 || VT == MVT::v8i16) &&
"Unexpected MVEEXT type");
18753 EVT ExtVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18755 auto Extend = [&](
SDValue V) {
18765 SDValue Ext = Extend(
N->getOperand(0));
18770 if (
auto *SVN = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(0))) {
18773 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
18778 auto CheckInregMask = [&](
int Start,
int Offset) {
18786 if (CheckInregMask(0, 0))
18788 else if (CheckInregMask(0, 1))
18789 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18790 else if (CheckInregMask(0, Mask.size()))
18792 else if (CheckInregMask(0, Mask.size() + 1))
18793 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18798 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18802 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18809 if (
N->getOperand(0)->getOpcode() ==
ISD::LOAD)
18819 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18820 int NumOuts =
N->getNumValues();
18821 assert((NumOuts == 2 || NumOuts == 4) &&
18822 "Expected 2 or 4 outputs to an MVEEXT");
18823 EVT LoadVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18825 if (
N->getNumOperands() == 4)
18831 StackPtr, MPI,
Align(4));
18834 for (
int I = 0;
I < NumOuts;
I++) {
18836 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18837 DAG.
getConstant(
I * 16 / NumOuts,
DL, StackPtr.getValueType()));
18842 VT, Chain,
Ptr, MPI, LoadVT,
Align(4));
18851 switch (
N->getOpcode()) {
18969 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
18976 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
18987 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
18995 unsigned LowWidth =
N->getOperand(0).getValueType().getSizeInBits();
18997 unsigned HighWidth =
N->getOperand(1).getValueType().getSizeInBits();
19005 unsigned HighWidth =
N->getOperand(0).getValueType().getSizeInBits();
19007 unsigned LowWidth =
N->getOperand(1).getValueType().getSizeInBits();
19015 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19026 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19035 switch (
N->getConstantOperandVal(1)) {
19036 case Intrinsic::arm_neon_vld1:
19037 case Intrinsic::arm_neon_vld1x2:
19038 case Intrinsic::arm_neon_vld1x3:
19039 case Intrinsic::arm_neon_vld1x4:
19040 case Intrinsic::arm_neon_vld2:
19041 case Intrinsic::arm_neon_vld3:
19042 case Intrinsic::arm_neon_vld4:
19043 case Intrinsic::arm_neon_vld2lane:
19044 case Intrinsic::arm_neon_vld3lane:
19045 case Intrinsic::arm_neon_vld4lane:
19046 case Intrinsic::arm_neon_vld2dup:
19047 case Intrinsic::arm_neon_vld3dup:
19048 case Intrinsic::arm_neon_vld4dup:
19049 case Intrinsic::arm_neon_vst1:
19050 case Intrinsic::arm_neon_vst1x2:
19051 case Intrinsic::arm_neon_vst1x3:
19052 case Intrinsic::arm_neon_vst1x4:
19053 case Intrinsic::arm_neon_vst2:
19054 case Intrinsic::arm_neon_vst3:
19055 case Intrinsic::arm_neon_vst4:
19056 case Intrinsic::arm_neon_vst2lane:
19057 case Intrinsic::arm_neon_vst3lane:
19058 case Intrinsic::arm_neon_vst4lane:
19060 case Intrinsic::arm_mve_vld2q:
19061 case Intrinsic::arm_mve_vld4q:
19062 case Intrinsic::arm_mve_vst2q:
19063 case Intrinsic::arm_mve_vst4q:
19080 unsigned *
Fast)
const {
19089 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19091 if (AllowsUnaligned) {
19093 *
Fast = Subtarget->hasV7Ops();
19098 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19102 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->
isLittle())) {
19109 if (!Subtarget->hasMVEIntegerOps())
19113 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19114 Ty == MVT::v2i1)) {
19122 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19138 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19139 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19140 Ty == MVT::v2f64) {
19153 if ((
Op.isMemcpy() ||
Op.isZeroMemset()) && Subtarget->hasNEON() &&
19154 !FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat)) {
19156 if (
Op.size() >= 16 &&
19162 }
else if (
Op.size() >= 8 &&
19183 return (SrcBits == 64 && DestBits == 32);
19192 return (SrcBits == 64 && DestBits == 32);
19228 return Subtarget->hasFullFP16();
19238 return Ext->getType()->getScalarSizeInBits() ==
19239 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
19244 !areExtDoubled(cast<Instruction>(Ext1)) ||
19245 !areExtDoubled(cast<Instruction>(Ext2)))
19256 if (!
I->getType()->isVectorTy())
19259 if (Subtarget->hasNEON()) {
19260 switch (
I->getOpcode()) {
19261 case Instruction::Sub:
19262 case Instruction::Add: {
19274 if (!Subtarget->hasMVEIntegerOps())
19278 if (!
I->hasOneUse())
19280 auto *Sub = cast<Instruction>(*
I->users().begin());
19281 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(1) ==
I;
19291 switch (
I->getOpcode()) {
19292 case Instruction::Add:
19293 case Instruction::Mul:
19294 case Instruction::FAdd:
19295 case Instruction::ICmp:
19296 case Instruction::FCmp:
19298 case Instruction::FMul:
19299 return !IsFMSMul(
I);
19300 case Instruction::Sub:
19301 case Instruction::FSub:
19302 case Instruction::Shl:
19303 case Instruction::LShr:
19304 case Instruction::AShr:
19305 return Operand == 1;
19306 case Instruction::Call:
19307 if (
auto *II = dyn_cast<IntrinsicInst>(
I)) {
19308 switch (II->getIntrinsicID()) {
19309 case Intrinsic::fma:
19311 case Intrinsic::sadd_sat:
19312 case Intrinsic::uadd_sat:
19313 case Intrinsic::arm_mve_add_predicated:
19314 case Intrinsic::arm_mve_mul_predicated:
19315 case Intrinsic::arm_mve_qadd_predicated:
19316 case Intrinsic::arm_mve_vhadd:
19317 case Intrinsic::arm_mve_hadd_predicated:
19318 case Intrinsic::arm_mve_vqdmull:
19319 case Intrinsic::arm_mve_vqdmull_predicated:
19320 case Intrinsic::arm_mve_vqdmulh:
19321 case Intrinsic::arm_mve_qdmulh_predicated:
19322 case Intrinsic::arm_mve_vqrdmulh:
19323 case Intrinsic::arm_mve_qrdmulh_predicated:
19324 case Intrinsic::arm_mve_fma_predicated:
19326 case Intrinsic::ssub_sat:
19327 case Intrinsic::usub_sat:
19328 case Intrinsic::arm_mve_sub_predicated:
19329 case Intrinsic::arm_mve_qsub_predicated:
19330 case Intrinsic::arm_mve_hsub_predicated:
19331 case Intrinsic::arm_mve_vhsub:
19332 return Operand == 1;
19343 for (
auto OpIdx :
enumerate(
I->operands())) {
19344 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
19346 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
19350 if (Shuffle->
getOpcode() == Instruction::BitCast)
19351 Shuffle = dyn_cast<Instruction>(Shuffle->
getOperand(0));
19358 if (!IsSinker(
I, OpIdx.index()))
19363 for (
Use &U :
Op->uses()) {
19365 if (!IsSinker(
Insn, U.getOperandNo()))
19378 if (!Subtarget->hasMVEIntegerOps())
19396 if (
auto *Ld = dyn_cast<MaskedLoadSDNode>(ExtVal.
getOperand(0))) {
19397 if (Ld->isExpandingLoad())
19401 if (Subtarget->hasMVEIntegerOps())
19446bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(
const MachineFunction &MF,
19454 return Subtarget->hasMVEFloatOps();
19472 unsigned Scale = 1;
19489 if ((V & (Scale - 1)) != 0)
19491 return isUInt<5>(V / Scale);
19498 if (VT.
isVector() && Subtarget->hasNEON())
19501 !Subtarget->hasMVEFloatOps())
19504 bool IsNeg =
false;
19510 unsigned NumBytes = std::max((
unsigned)VT.
getSizeInBits() / 8, 1U);
19513 if (VT.
isVector() && Subtarget->hasMVEIntegerOps()) {
19517 return isShiftedUInt<7,2>(V);
19520 return isShiftedUInt<7,1>(V);
19522 return isUInt<7>(V);
19529 if (VT.
isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19530 return isShiftedUInt<8, 1>(V);
19533 return isShiftedUInt<8, 2>(V);
19535 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19538 return isUInt<8>(V);
19539 return isUInt<12>(V);
19565 default:
return false;
19570 return isUInt<12>(V);
19573 return isUInt<8>(V);
19578 return isShiftedUInt<8, 2>(V);
19584 int Scale = AM.
Scale;
19589 default:
return false;
19597 Scale = Scale & ~1;
19598 return Scale == 2 || Scale == 4 || Scale == 8;
19615 if (Scale & 1)
return false;
19622 const int Scale = AM.
Scale;
19632 return (Scale == 1) || (!AM.
HasBaseReg && Scale == 2);
19648 switch (AM.
Scale) {
19665 int Scale = AM.
Scale;
19667 default:
return false;
19671 if (Scale < 0) Scale = -Scale;
19679 if (Scale == 1 || (AM.
HasBaseReg && Scale == -1))
19692 if (Scale & 1)
return false;
19705 if (!Subtarget->isThumb())
19712 return Imm >= 0 && Imm <= 255;
19721 int64_t AbsImm = std::abs(Imm);
19722 if (!Subtarget->isThumb())
19727 return AbsImm >= 0 && AbsImm <= 255;
19762 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
19766 int RHSC = (int)
RHS->getZExtValue();
19767 if (RHSC < 0 && RHSC > -256) {
19777 }
else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
19780 int RHSC = (int)
RHS->getZExtValue();
19781 if (RHSC < 0 && RHSC > -0x1000) {
19823 int RHSC = (int)
RHS->getZExtValue();
19824 if (RHSC < 0 && RHSC > -0x100) {
19829 }
else if (RHSC > 0 && RHSC < 0x100) {
19840 bool isSEXTLoad,
bool IsMasked,
bool isLE,
19845 if (!isa<ConstantSDNode>(
Ptr->getOperand(1)))
19851 bool CanChangeType = isLE && !IsMasked;
19854 int RHSC = (int)
RHS->getZExtValue();
19856 auto IsInRange = [&](
int RHSC,
int Limit,
int Scale) {
19857 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
19862 }
else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
19873 if (VT == MVT::v4i16) {
19874 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
19876 }
else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
19877 if (IsInRange(RHSC, 0x80, 1))
19879 }
else if (Alignment >= 4 &&
19880 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
19881 IsInRange(RHSC, 0x80, 4))
19883 else if (Alignment >= 2 &&
19884 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
19885 IsInRange(RHSC, 0x80, 2))
19887 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
19906 bool isSEXTLoad =
false;
19907 bool IsMasked =
false;
19909 Ptr = LD->getBasePtr();
19910 VT = LD->getMemoryVT();
19911 Alignment = LD->getAlign();
19913 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
19914 Ptr = ST->getBasePtr();
19915 VT = ST->getMemoryVT();
19916 Alignment = ST->getAlign();
19918 Ptr = LD->getBasePtr();
19919 VT = LD->getMemoryVT();
19920 Alignment = LD->getAlign();
19924 Ptr = ST->getBasePtr();
19925 VT = ST->getMemoryVT();
19926 Alignment = ST->getAlign();
19932 bool isLegal =
false;
19934 isLegal = Subtarget->hasMVEIntegerOps() &&
19936 Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
19964 bool isSEXTLoad =
false, isNonExt;
19965 bool IsMasked =
false;
19967 VT = LD->getMemoryVT();
19968 Ptr = LD->getBasePtr();
19969 Alignment = LD->getAlign();
19972 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
19973 VT = ST->getMemoryVT();
19974 Ptr = ST->getBasePtr();
19975 Alignment = ST->getAlign();
19976 isNonExt = !ST->isTruncatingStore();
19978 VT = LD->getMemoryVT();
19979 Ptr = LD->getBasePtr();
19980 Alignment = LD->getAlign();
19985 VT = ST->getMemoryVT();
19986 Ptr = ST->getBasePtr();
19987 Alignment = ST->getAlign();
19988 isNonExt = !ST->isTruncatingStore();
19996 assert(
Op->getValueType(0) == MVT::i32 &&
"Non-i32 post-inc op?!");
19997 if (
Op->getOpcode() !=
ISD::ADD || !isNonExt)
19999 auto *
RHS = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
20000 if (!
RHS ||
RHS->getZExtValue() != 4)
20002 if (Alignment <
Align(4))
20006 Base =
Op->getOperand(0);
20012 bool isLegal =
false;
20014 isLegal = Subtarget->hasMVEIntegerOps() &&
20047 const APInt &DemandedElts,
20049 unsigned Depth)
const {
20052 switch (
Op.getOpcode()) {
20059 if (
Op.getResNo() == 0) {
20085 case Intrinsic::arm_ldaex:
20086 case Intrinsic::arm_ldrex: {
20087 EVT VT = cast<MemIntrinsicSDNode>(
Op)->getMemoryVT();
20101 const APInt &Mask =
Op.getConstantOperandAPInt(2);
20102 Known.
Zero &= Mask;
20108 const SDValue &SrcSV =
Op.getOperand(0);
20114 "VGETLANE index out of bounds");
20119 EVT VT =
Op.getValueType();
20126 Known = Known.
sext(DstSz);
20128 Known = Known.
zext(DstSz);
20136 Known = KnownOp.
zext(32);
20151 true,
false,
false, KnownOp1,
20177 EVT VT =
Op.getValueType();
20183 assert(VT == MVT::i32 &&
"Unexpected integer type");
20190 unsigned Mask =
C->getZExtValue();
20193 unsigned ShrunkMask = Mask & Demanded;
20194 unsigned ExpandedMask = Mask | ~Demanded;
20198 if (ShrunkMask == 0)
20204 if (ExpandedMask == ~0U)
20207 auto IsLegalMask = [ShrunkMask, ExpandedMask](
unsigned Mask) ->
bool {
20208 return (ShrunkMask & Mask) == ShrunkMask && (~ExpandedMask & Mask) == 0;
20210 auto UseMask = [Mask,
Op, VT, &TLO](
unsigned NewMask) ->
bool {
20211 if (NewMask == Mask)
20220 if (IsLegalMask(0xFF))
20221 return UseMask(0xFF);
20224 if (IsLegalMask(0xFFFF))
20225 return UseMask(0xFFFF);
20229 if (ShrunkMask < 256)
20230 return UseMask(ShrunkMask);
20234 if ((
int)ExpandedMask <= -2 && (
int)ExpandedMask >= -256)
20235 return UseMask(ExpandedMask);
20250 unsigned Depth)
const {
20251 unsigned Opc =
Op.getOpcode();
20259 if (
Op.getResNo() == 0 && !
Op->hasAnyUseOfValue(1) &&
20260 isa<ConstantSDNode>(
Op->getOperand(2))) {
20261 unsigned ShAmt =
Op->getConstantOperandVal(2);
20273 unsigned ModImm =
Op.getConstantOperandVal(1);
20274 unsigned EltBits = 0;
20276 if ((OriginalDemandedBits & Mask) == 0)
20282 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO,
Depth);
20291 if (!Subtarget->hasV6Ops())
20297 SplitString(AsmStr, AsmPieces,
";\n");
20299 switch (AsmPieces.
size()) {
20300 default:
return false;
20302 AsmStr = AsmPieces[0];
20304 SplitString(AsmStr, AsmPieces,
" \t,");
20307 if (AsmPieces.
size() == 3 &&
20308 AsmPieces[0] ==
"rev" && AsmPieces[1] ==
"$0" && AsmPieces[2] ==
"$1" &&
20309 IA->getConstraintString().compare(0, 4,
"=l,l") == 0) {
20332 if (ConstraintVT.
isVector() && Subtarget->hasNEON() &&
20344 unsigned S = Constraint.
size();
20346 switch (Constraint[0]) {
20358 }
else if (S == 2) {
20359 switch (Constraint[0]) {
20376 Value *CallOperandVal =
info.CallOperandVal;
20379 if (!CallOperandVal)
20383 switch (*constraint) {
20389 if (Subtarget->isThumb())
20403using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20407 switch (Constraint.
size()) {
20410 switch (Constraint[0]) {
20412 if (Subtarget->isThumb())
20413 return RCPair(0U, &ARM::tGPRRegClass);
20414 return RCPair(0U, &ARM::GPRRegClass);
20416 if (Subtarget->isThumb())
20417 return RCPair(0U, &ARM::hGPRRegClass);
20421 return RCPair(0U, &ARM::tGPRRegClass);
20422 return RCPair(0U, &ARM::GPRRegClass);
20424 if (VT == MVT::Other)
20426 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20427 return RCPair(0U, &ARM::SPRRegClass);
20429 return RCPair(0U, &ARM::DPRRegClass);
20431 return RCPair(0U, &ARM::QPRRegClass);
20434 if (VT == MVT::Other)
20436 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20437 return RCPair(0U, &ARM::SPR_8RegClass);
20439 return RCPair(0U, &ARM::DPR_8RegClass);
20441 return RCPair(0U, &ARM::QPR_8RegClass);
20444 if (VT == MVT::Other)
20446 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20447 return RCPair(0U, &ARM::SPRRegClass);
20449 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20451 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20457 if (Constraint[0] ==
'T') {
20458 switch (Constraint[1]) {
20462 return RCPair(0U, &ARM::tGPREvenRegClass);
20464 return RCPair(0U, &ARM::tGPROddRegClass);
20473 if (
StringRef(
"{cc}").equals_insensitive(Constraint))
20474 return std::make_pair(
unsigned(ARM::CPSR), &ARM::CCRRegClass);
20483 std::vector<SDValue> &Ops,
20488 if (Constraint.
size() != 1)
20491 char ConstraintLetter = Constraint[0];
20492 switch (ConstraintLetter) {
20495 case 'I':
case 'J':
case 'K':
case 'L':
20496 case 'M':
case 'N':
case 'O':
20501 int64_t CVal64 =
C->getSExtValue();
20502 int CVal = (int) CVal64;
20505 if (CVal != CVal64)
20508 switch (ConstraintLetter) {
20512 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
20513 if (CVal >= 0 && CVal <= 65535)
20520 if (CVal >= 0 && CVal <= 255)
20522 }
else if (Subtarget->
isThumb2()) {
20541 if (CVal >= -255 && CVal <= -1)
20547 if (CVal >= -4095 && CVal <= 4095)
20560 }
else if (Subtarget->
isThumb2()) {
20583 if (CVal >= -7 && CVal < 7)
20585 }
else if (Subtarget->
isThumb2()) {
20608 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
20614 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
20622 if (CVal >= 0 && CVal <= 31)
20631 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
20640 if (Result.getNode()) {
20641 Ops.push_back(Result);
20651 "Unhandled Opcode in getDivRemLibcall");
20657 case MVT::i8: LC =
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
20658 case MVT::i16: LC =
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
20659 case MVT::i32: LC =
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
20660 case MVT::i64: LC =
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
20669 "Unhandled Opcode in getDivRemArgList");
20674 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
20675 EVT ArgVT =
N->getOperand(i).getValueType();
20677 Entry.Node =
N->getOperand(i);
20681 Args.push_back(Entry);
20692 "Register-based DivRem lowering only");
20693 unsigned Opcode =
Op->getOpcode();
20695 "Invalid opcode for Div/Rem lowering");
20697 EVT VT =
Op->getValueType(0);
20700 if (VT == MVT::i64 && isa<ConstantSDNode>(
Op.getOperand(1))) {
20719 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
20720 : Subtarget->hasDivideInARMMode();
20721 if (hasDivide &&
Op->getValueType(0).isSimple() &&
20722 Op->getSimpleValueType(0) == MVT::i32) {
20724 const SDValue Dividend =
Op->getOperand(0);
20725 const SDValue Divisor =
Op->getOperand(1);
20726 SDValue Div = DAG.
getNode(DivOpcode, dl, VT, Dividend, Divisor);
20730 SDValue Values[2] = {Div, Rem};
20762 EVT VT =
N->getValueType(0);
20764 if (VT == MVT::i64 && isa<ConstantSDNode>(
N->getOperand(1))) {
20768 Result[0], Result[1]);
20772 std::vector<Type*> RetTyParams;
20773 Type *RetTyElement;
20783 RetTyParams.push_back(RetTyElement);
20784 RetTyParams.push_back(RetTyElement);
20801 CallLoweringInfo CLI(DAG);
20805 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
20808 SDNode *ResNode = CallResult.first.getNode();
20823 "no-stack-arg-probe")) {
20825 cast<ConstantSDNode>(
Op.getOperand(2))->getMaybeAlignValue();
20834 SDValue Ops[2] = { SP, Chain };
20851 SDValue Ops[2] = { NewSP, Chain };
20856 bool IsStrict =
Op->isStrictFPOpcode();
20857 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20858 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20860 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
20861 "Unexpected type for custom-lowering FP_EXTEND");
20864 "With both FP DP and 16, any FP conversion is legal!");
20866 assert(!(DstSz == 32 && Subtarget->hasFP16()) &&
20867 "With FP16, 16 to 32 conversion is legal!");
20870 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) {
20875 Loc,
Op.getValueType(), SrcVal);
20888 MakeLibCallOptions CallOptions;
20890 for (
unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
20891 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
20892 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
20893 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
20897 {DstVT, MVT::Other}, {Chain, SrcVal});
20904 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20905 "Unexpected type for custom-lowering FP_EXTEND");
20906 std::tie(SrcVal, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20911 return IsStrict ? DAG.
getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
20915 bool IsStrict =
Op->isStrictFPOpcode();
20917 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20919 EVT DstVT =
Op.getValueType();
20920 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20923 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
20924 "Unexpected type for custom-lowering FP_ROUND");
20927 "With both FP DP and 16, any FP conversion is legal!");
20932 if (SrcSz == 32 && Subtarget->hasFP16())
20937 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20938 "Unexpected type for custom-lowering FP_ROUND");
20939 MakeLibCallOptions CallOptions;
20942 std::tie(Result, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20954 if (v == 0xffffffff)
20966 bool ForCodeSize)
const {
20969 if (VT == MVT::f16 && Subtarget->hasFullFP16())
20971 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
20974 if (VT == MVT::f32)
20976 if (VT == MVT::f64 && Subtarget->hasFP64())
20987 unsigned Intrinsic)
const {
20988 switch (Intrinsic) {
20989 case Intrinsic::arm_neon_vld1:
20990 case Intrinsic::arm_neon_vld2:
20991 case Intrinsic::arm_neon_vld3:
20992 case Intrinsic::arm_neon_vld4:
20993 case Intrinsic::arm_neon_vld2lane:
20994 case Intrinsic::arm_neon_vld3lane:
20995 case Intrinsic::arm_neon_vld4lane:
20996 case Intrinsic::arm_neon_vld2dup:
20997 case Intrinsic::arm_neon_vld3dup:
20998 case Intrinsic::arm_neon_vld4dup: {
21001 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21002 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21004 Info.ptrVal =
I.getArgOperand(0);
21006 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21007 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21012 case Intrinsic::arm_neon_vld1x2:
21013 case Intrinsic::arm_neon_vld1x3:
21014 case Intrinsic::arm_neon_vld1x4: {
21017 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21018 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21020 Info.ptrVal =
I.getArgOperand(
I.arg_size() - 1);
21022 Info.align.reset();
21027 case Intrinsic::arm_neon_vst1:
21028 case Intrinsic::arm_neon_vst2:
21029 case Intrinsic::arm_neon_vst3:
21030 case Intrinsic::arm_neon_vst4:
21031 case Intrinsic::arm_neon_vst2lane:
21032 case Intrinsic::arm_neon_vst3lane:
21033 case Intrinsic::arm_neon_vst4lane: {
21036 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21037 unsigned NumElts = 0;
21038 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21039 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21042 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21045 Info.ptrVal =
I.getArgOperand(0);
21047 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21048 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21053 case Intrinsic::arm_neon_vst1x2:
21054 case Intrinsic::arm_neon_vst1x3:
21055 case Intrinsic::arm_neon_vst1x4: {
21058 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21059 unsigned NumElts = 0;
21060 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21061 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21064 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21067 Info.ptrVal =
I.getArgOperand(0);
21069 Info.align.reset();
21074 case Intrinsic::arm_mve_vld2q:
21075 case Intrinsic::arm_mve_vld4q: {
21078 Type *VecTy = cast<StructType>(
I.getType())->getElementType(1);
21079 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21081 Info.ptrVal =
I.getArgOperand(0);
21088 case Intrinsic::arm_mve_vst2q:
21089 case Intrinsic::arm_mve_vst4q: {
21092 Type *VecTy =
I.getArgOperand(1)->getType();
21093 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21095 Info.ptrVal =
I.getArgOperand(0);
21102 case Intrinsic::arm_mve_vldr_gather_base:
21103 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21105 Info.ptrVal =
nullptr;
21111 case Intrinsic::arm_mve_vldr_gather_base_wb:
21112 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21114 Info.ptrVal =
nullptr;
21120 case Intrinsic::arm_mve_vldr_gather_offset:
21121 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21123 Info.ptrVal =
nullptr;
21125 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
21132 case Intrinsic::arm_mve_vstr_scatter_base:
21133 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21135 Info.ptrVal =
nullptr;
21141 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21142 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21144 Info.ptrVal =
nullptr;
21150 case Intrinsic::arm_mve_vstr_scatter_offset:
21151 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21153 Info.ptrVal =
nullptr;
21155 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(3))->getZExtValue();
21162 case Intrinsic::arm_ldaex:
21163 case Intrinsic::arm_ldrex: {
21164 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21165 Type *ValTy =
I.getParamElementType(0);
21168 Info.ptrVal =
I.getArgOperand(0);
21170 Info.align =
DL.getABITypeAlign(ValTy);
21174 case Intrinsic::arm_stlex:
21175 case Intrinsic::arm_strex: {
21176 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21177 Type *ValTy =
I.getParamElementType(1);
21180 Info.ptrVal =
I.getArgOperand(1);
21182 Info.align =
DL.getABITypeAlign(ValTy);
21186 case Intrinsic::arm_stlexd:
21187 case Intrinsic::arm_strexd:
21189 Info.memVT = MVT::i64;
21190 Info.ptrVal =
I.getArgOperand(2);
21196 case Intrinsic::arm_ldaexd:
21197 case Intrinsic::arm_ldrexd:
21199 Info.memVT = MVT::i64;
21200 Info.ptrVal =
I.getArgOperand(0);
21220 if (Bits == 0 || Bits > 32)
21226 unsigned Index)
const {
21238 if (!Subtarget->hasDataBarrier()) {
21242 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21279 if (Subtarget->preferISHSTBarriers())
21312 bool has64BitAtomicStore;
21314 has64BitAtomicStore =
false;
21315 else if (Subtarget->isThumb())
21316 has64BitAtomicStore = Subtarget->hasV7Ops();
21318 has64BitAtomicStore = Subtarget->hasV6Ops();
21320 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21334 bool has64BitAtomicLoad;
21336 has64BitAtomicLoad =
false;
21337 else if (Subtarget->isThumb())
21338 has64BitAtomicLoad = Subtarget->hasV7Ops();
21340 has64BitAtomicLoad = Subtarget->hasV6Ops();
21357 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21358 else if (Subtarget->isThumb())
21359 hasAtomicRMW = Subtarget->hasV7Ops();
21361 hasAtomicRMW = Subtarget->hasV6Ops();
21362 if (
Size <= (Subtarget->
isMClass() ? 32U : 64U) && hasAtomicRMW) {
21385 bool HasAtomicCmpXchg;
21387 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21388 else if (Subtarget->isThumb())
21389 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21391 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21393 HasAtomicCmpXchg &&
Size <= (Subtarget->
isMClass() ? 32U : 64U))
21400 return InsertFencesForAtomic;
21413 M.getOrInsertGlobal(
"__security_cookie",
21421 F->addParamAttr(0, Attribute::AttrKind::InReg);
21427 return M.getGlobalVariable(
"__security_cookie");
21434 return M.getFunction(
"__security_check_cookie");
21439 unsigned &
Cost)
const {
21441 if (!Subtarget->hasNEON())
21453 if (!isa<ConstantInt>(
Idx))
21468 return Subtarget->hasV6T2Ops();
21472 return Subtarget->hasV6T2Ops();
21477 if (!Subtarget->hasV7Ops())
21483 if (!Mask || Mask->getValue().getBitWidth() > 32u)
21485 auto MaskVal =
unsigned(Mask->getValue().getZExtValue());
21510 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
21522 Lo, Builder.
CreateShl(
Hi, ConstantInt::get(ValueTy, 32)),
"val64");
21525 Type *Tys[] = {
Addr->getType() };
21526 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
21531 0,
Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
21537 if (!Subtarget->hasV7Ops())
21554 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
21565 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
21566 Type *Tys[] = {
Addr->getType() };
21588 return (
DL.getTypeSizeInBits(VecTy) + 127) / 128;
21595 unsigned VecSize =
DL.getTypeSizeInBits(VecTy);
21598 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
21606 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
21614 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
21617 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
21622 if (Subtarget->hasNEON() && VecSize == 64)
21624 return VecSize % 128 == 0;
21628 if (Subtarget->hasNEON())
21630 if (Subtarget->hasMVEIntegerOps())
21650 "Invalid interleave factor");
21651 assert(!Shuffles.
empty() &&
"Empty shufflevector input");
21653 "Unmatched number of shufflevectors and indices");
21655 auto *VecTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21656 Type *EltTy = VecTy->getElementType();
21679 if (NumLoads > 1) {
21683 VecTy->getNumElements() / NumLoads);
21688 auto createLoadIntrinsic = [&](
Value *BaseAddr) {
21689 if (Subtarget->hasNEON()) {
21691 Type *Tys[] = {VecTy, PtrTy};
21692 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
21693 Intrinsic::arm_neon_vld3,
21694 Intrinsic::arm_neon_vld4};
21702 return Builder.
CreateCall(VldnFunc, Ops,
"vldN");
21704 assert((Factor == 2 || Factor == 4) &&
21705 "expected interleave factor of 2 or 4 for MVE");
21707 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
21709 Type *Tys[] = {VecTy, PtrTy};
21715 return Builder.
CreateCall(VldnFunc, Ops,
"vldN");
21724 for (
unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
21729 VecTy->getNumElements() * Factor);
21731 CallInst *VldN = createLoadIntrinsic(BaseAddr);
21735 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21737 unsigned Index = Indices[i];
21747 SubVecs[SV].push_back(SubVec);
21756 auto &SubVec = SubVecs[SVI];
21759 SVI->replaceAllUsesWith(WideVec);
21793 unsigned Factor)
const {
21795 "Invalid interleave factor");
21797 auto *VecTy = cast<FixedVectorType>(SVI->
getType());
21798 assert(VecTy->getNumElements() % Factor == 0 &&
"Invalid interleaved store");
21800 unsigned LaneLen = VecTy->getNumElements() / Factor;
21801 Type *EltTy = VecTy->getElementType();
21804 const DataLayout &
DL = SI->getModule()->getDataLayout();
21805 Align Alignment = SI->getAlign();
21822 Type *IntTy =
DL.getIntPtrType(EltTy);
21834 Value *BaseAddr = SI->getPointerOperand();
21836 if (NumStores > 1) {
21839 LaneLen /= NumStores;
21847 auto createStoreIntrinsic = [&](
Value *BaseAddr,
21849 if (Subtarget->hasNEON()) {
21850 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
21851 Intrinsic::arm_neon_vst3,
21852 Intrinsic::arm_neon_vst4};
21853 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21854 Type *Tys[] = {PtrTy, SubVecTy};
21857 SI->getModule(), StoreInts[Factor - 2], Tys);
21865 assert((Factor == 2 || Factor == 4) &&
21866 "expected interleave factor of 2 or 4 for MVE");
21868 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
21869 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21870 Type *Tys[] = {PtrTy, SubVecTy};
21877 for (
unsigned F = 0;
F < Factor;
F++) {
21885 for (
unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
21888 if (StoreCount > 0)
21890 BaseAddr, LaneLen * Factor);
21895 for (
unsigned i = 0; i < Factor; i++) {
21896 unsigned IdxI = StoreCount * LaneLen * Factor + i;
21897 if (Mask[IdxI] >= 0) {
21901 unsigned StartMask = 0;
21902 for (
unsigned j = 1; j < LaneLen; j++) {
21903 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
21904 if (Mask[IdxJ * Factor + IdxI] >= 0) {
21905 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
21920 createStoreIntrinsic(BaseAddr, Shuffles);
21935 if (
auto *ST = dyn_cast<StructType>(Ty)) {
21936 for (
unsigned i = 0; i < ST->getNumElements(); ++i) {
21940 Members += SubMembers;
21942 }
else if (
auto *AT = dyn_cast<ArrayType>(Ty)) {
21946 Members += SubMembers * AT->getNumElements();
21957 }
else if (
auto *VT = dyn_cast<VectorType>(Ty)) {
21964 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
21966 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
21968 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
21981 return (Members > 0 && Members <= 4);
21987 const Align ABITypeAlign =
DL.getABITypeAlign(ArgTy);
21989 return ABITypeAlign;
21993 return std::min(ABITypeAlign,
DL.getStackAlignment());
22002 if (getEffectiveCallingConv(CallConv, isVarArg) !=
22012 return IsHA || IsIntArray;
22016 const Constant *PersonalityFn)
const {
22023 const Constant *PersonalityFn)
const {
22035void ARMTargetLowering::insertCopiesSplitCSR(
22039 const MCPhysReg *IStart =
TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
22049 RC = &ARM::GPRRegClass;
22050 else if (ARM::DPRRegClass.
contains(*
I))
22051 RC = &ARM::DPRRegClass;
22061 assert(Entry->getParent()->getFunction().hasFnAttribute(
22062 Attribute::NoUnwind) &&
22063 "Function should be nounwind in insertCopiesSplitCSR!");
22064 Entry->addLiveIn(*
I);
22069 for (
auto *Exit : Exits)
22071 TII->get(TargetOpcode::COPY), *
I)
22082 return Subtarget->hasMVEIntegerOps();
22087 auto *VTy = dyn_cast<FixedVectorType>(Ty);
22092 unsigned NumElements = VTy->getNumElements();
22099 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22100 return Subtarget->hasMVEFloatOps();
22105 return Subtarget->hasMVEIntegerOps() &&
22106 (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
22107 ScalarTy->isIntegerTy(32));
22119 assert(TyWidth >= 128 &&
"Width of vector type must be at least 128 bits");
22121 if (TyWidth > 128) {
22126 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22128 auto *LowerSplitA =
B.CreateShuffleVector(InputA, LowerSplitMask);
22129 auto *LowerSplitB =
B.CreateShuffleVector(InputB, LowerSplitMask);
22130 auto *UpperSplitA =
B.CreateShuffleVector(InputA, UpperSplitMask);
22131 auto *UpperSplitB =
B.CreateShuffleVector(InputB, UpperSplitMask);
22132 Value *LowerSplitAcc =
nullptr;
22133 Value *UpperSplitAcc =
nullptr;
22136 LowerSplitAcc =
B.CreateShuffleVector(
Accumulator, LowerSplitMask);
22137 UpperSplitAcc =
B.CreateShuffleVector(
Accumulator, UpperSplitMask);
22141 B, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
22143 B, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);
22146 return B.CreateShuffleVector(LowerSplitInt, UpperSplitInt, JoinMask);
22153 ConstRotation = ConstantInt::get(IntTy, (
int)Rotation);
22156 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmlaq, Ty,
22158 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmulq, Ty,
22159 {ConstRotation, InputB, InputA});
22164 auto *ConstHalving = ConstantInt::get(IntTy, 1);
22167 ConstRotation = ConstantInt::get(IntTy, 0);
22169 ConstRotation = ConstantInt::get(IntTy, 1);
22171 if (!ConstRotation)
22174 return B.CreateIntrinsic(Intrinsic::arm_mve_vcaddq, Ty,
22175 {ConstHalving, ConstRotation, InputA, InputB});
unsigned const MachineRegisterInfo * MRI
static bool isAddSubSExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG, bool isSigned)
static bool isZeroExtended(SDValue N, SelectionDAG &DAG)
static bool areExtractExts(Value *Ext1, Value *Ext2)
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static const MCPhysReg GPRArgRegs[]
static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1, SDValue V2, unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
static bool isSignExtended(SDValue N, SelectionDAG &DAG)
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
static bool isAddSubZExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG)
static bool isStore(int Opcode)
static bool isThumb(const MCSubtargetInfo &STI)
static SDValue LowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, const TargetInstrInfo *TII)
MatchingStackOffset - Return true if the given stack call argument is already available in the same p...
static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG)
static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total value size to 64 bits.
static cl::opt< unsigned > ConstpoolPromotionMaxSize("arm-promote-constant-max-size", cl::Hidden, cl::desc("Maximum size of constant to promote into a constant pool"), cl::init(64))
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isVTBLMask(ArrayRef< int > M, EVT VT)
static SDValue PerformSUBCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
static cl::opt< bool > EnableConstpoolPromotion("arm-promote-constant", cl::Hidden, cl::desc("Enable / disable promotion of unnamed_addr constants into " "constant pools"), cl::init(false))
static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG)
static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
static SDValue PerformExtractEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static const APInt * isPowerOf2Constant(SDValue V)
static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG)
static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static bool isValidMVECond(unsigned CC, bool IsFloat)
static SDValue PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC)
IntCCToARMCC - Convert a DAG integer condition code to an ARM CC.
static SDValue PerformSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSTORECombine - Target-specific dag combine xforms for ISD::STORE.
static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry, SelectionDAG &DAG)
static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isGTorGE(ISD::CondCode CC)
static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a vldN-lane (N > 1) intrinsic,...
static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask)
static bool isReverseMask(ArrayRef< int > M, EVT VT)
static bool isVZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of "vector_shuffle v,...
static SDValue PerformSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG)
static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc)
static bool isVTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool CanInvertMVEVCMP(SDValue N)
static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG)
static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformShiftCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
PerformShiftCombine - Checks for immediate versions of vector shifts and lowers them.
static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, ARMCC::CondCodes &CondCode2)
FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static EVT getVectorTyFromPredicateVector(EVT VT)
static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
static bool isSRL16(const SDValue &Op)
static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC)
static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr, SDValue Inc, const SelectionDAG &DAG)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static Register genTPEntry(MachineBasicBlock *TpEntry, MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpExit, Register OpSizeReg, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI)
Adds logic in loop entry MBB to calculate loop iteration count and adds t2WhileLoopSetup and t2WhileL...
static bool isLTorLE(ISD::CondCode CC)
static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl, SelectionDAG &DAG)
static SDValue PerformBITCASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG)
static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG)
static bool hasNormalLoadOperand(SDNode *N)
hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node are normal,...
static SDValue PerformInsertEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
PerformInsertEltCombine - Target-specific dag combine xforms for ISD::INSERT_VECTOR_ELT.
static SDValue PerformVDUPLANECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVDUPLANECombine - Target-specific dag combine xforms for ARMISD::VDUPLANE.
static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static cl::opt< unsigned > ConstpoolPromotionMaxTotal("arm-promote-constant-max-total", cl::Hidden, cl::desc("Maximum size of ALL constants to promote into a constant pool"), cl::init(128))
static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static RTLIB::Libcall getDivRemLibcall(const SDNode *N, MVT::SimpleValueType SVT)
static SDValue PerformABSCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG &DAG)
SkipLoadExtensionForVMULL - return a load of the original vector size that does not do any sign/zero ...
static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static const MCPhysReg GPRArgRegs[]
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombineWithOperands - Try DAG combinations for an ADD with operands N0 and N1.
static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT, SelectionDAG &DAG)
static bool isVZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformORCombineToSMULWBT(SDNode *OR, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isVTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of "vector_shuffle v,...
static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue FindBFIToCombineWith(SDNode *N)
static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT, SelectionDAG &DAG)
static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, bool &swpCmpOps, bool &swpVselOps)
static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isS16(const SDValue &Op, SelectionDAG &DAG)
static bool isSRA16(const SDValue &Op)
static SDValue AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue LowerInterruptReturn(SmallVectorImpl< SDValue > &RetOps, const SDLoc &DL, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl, SelectionDAG &DAG)
static void expandf64Toi32(SDValue Op, SelectionDAG &DAG, SDValue &RetVal1, SDValue &RetVal2)
static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isSHL16(const SDValue &Op)
static bool isVEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseVEXT, unsigned &Imm)
static SDValue PerformMVEVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isTruncMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue PerformADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2)
Return the load opcode for a given load size.
static bool isLegalT2AddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
static bool isLegalMVEShuffleOp(unsigned PFEntry)
static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N, SelectionDAG &DAG)
static bool isVUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG)
PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for ISD::VECTOR_SHUFFLE.
static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG)
SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND, ANY_EXTEND,...
static bool isVMOVNTruncMask(ArrayRef< int > M, EVT ToVT, bool rev)
static SDValue PerformVQMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static MachineBasicBlock * OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ)
static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformAddcSubcCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static TargetLowering::ArgListTy getDivRemArgList(const SDNode *N, LLVMContext *Context, const ARMSubtarget *Subtarget)
static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl)
getZeroVector - Returns a vector of specified type with all zero elements.
static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static ARMCC::CondCodes getVCMPCondCode(SDValue N)
static cl::opt< bool > ARMInterworking("arm-interworking", cl::Hidden, cl::desc("Enable / disable ARM interworking (for debugging only)"), cl::init(true))
static void ReplaceREADCYCLECOUNTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformORCombineToBFI(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, SDValue &CC, bool &Invert, SDValue &OtherOp, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVSetCCToVCTPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isZeroVector(SDValue N)
static SDValue PerformAddeSubeCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void ReplaceCMP_SWAP_64Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isLowerSaturate(const SDValue LHS, const SDValue RHS, const SDValue TrueVal, const SDValue FalseVal, const ISD::CondCode CC, const SDValue K)
static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG)
static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned StSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment store operation with given size.
static bool isVMOVNMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue CombineBaseUpdate(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineBaseUpdate - Target-specific DAG combine function for VLDDUP, NEON load/store intrinsics,...
static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVRRDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMOVRRDCombine - Target-specific dag combine xforms for ARMISD::VMOVRRD.
static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain)
static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue PerformVMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMULCombine Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the special multi...
static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG)
static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformORCombine - Target-specific dag combine xforms for ISD::OR.
static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG)
static SDValue PerformTruncatingStoreCombine(StoreSDNode *St, SelectionDAG &DAG)
static unsigned SelectPairHalf(unsigned Elements, ArrayRef< int > Mask, unsigned Index)
static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned LdSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment load operation with given size.
static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG)
static bool isValidBaseUpdate(SDNode *N, SDNode *User)
static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG, const ARMSubtarget *ST, const SDLoc &dl)
static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op)
static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformXORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment, bool isSEXTLoad, bool IsMasked, bool isLE, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
std::pair< unsigned, const TargetRegisterClass * > RCPair
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,...
static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
cl::opt< unsigned > MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden, cl::desc("Maximum interleave factor for MVE VLDn to generate."), cl::init(2))
static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, unsigned SplatBitSize, SelectionDAG &DAG, const SDLoc &dl, EVT &VT, EVT VectorVT, VMOVModImmType type)
isVMOVModifiedImm - Check if the specified splat value corresponds to a valid vector constant for a N...
static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC, SelectionDAG &DAG)
BC is a bitcast that is about to be turned into a VMOVDRR.
static SDValue promoteToConstantPool(const ARMTargetLowering *TLI, const GlobalValue *GV, SelectionDAG &DAG, EVT PtrVT, const SDLoc &dl)
static unsigned isNEONTwoResultShuffleMask(ArrayRef< int > ShuffleMask, EVT VT, unsigned &WhichResult, bool &isV_UNDEF)
Check if ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN), and return the corresponding AR...
static bool BitsProperlyConcatenate(const APInt &A, const APInt &B)
static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG)
static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target, struct BaseUpdateUser &User, bool SimpleConstIncOnly, TargetLowering::DAGCombinerInfo &DCI)
static bool allUsersAreInFunction(const Value *V, const Function *F)
Return true if all users of V are within function F, looking through ConstantExprs.
static bool isSingletonVEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG)
PerformVMOVDRRCombine - Target-specific dag combine xforms for ARMISD::VMOVDRR.
static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, SDValue &SatK)
static bool isLegalAddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
isLegalAddressImmediate - Return true if the integer value can be used as the offset of the target ad...
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isLegalT1AddressImmediate(int64_t V, EVT VT)
static SDValue CombineANDShift(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSHLSimplify(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformADDECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDECombine - Target-specific dag combine transform from ARMISD::ADDC, ARMISD::ADDE,...
static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformHWLoopCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool isVUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of "vector_shuffle v,...
static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base, uint64_t &Members)
static SDValue PerformMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
static SDValue PerformVDIVCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDIVCombine - VCVT (fixed-point to floating-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG)
static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm, bool &Negate)
static bool canChangeToInt(SDValue Op, bool &SeenZero, const ARMSubtarget *Subtarget)
canChangeToInt - Given the fp compare operand, return true if it is suitable to morph to an integer c...
static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2)
Return the store opcode for a given store size.
static bool IsVUZPShuffleNode(SDNode *N)
static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget, MachineInstr &MI, const SDNode *Node)
Attaches vregs to MEMCPY that it will use as scratch registers when it is expanded into LDM/STM.
static bool isFloatingPointZero(SDValue Op)
isFloatingPointZero - Return true if this is +0.0.
static SDValue findMUL_LOHI(SDValue V)
static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N, SelectionDAG &DAG)
static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG)
static void genTPLoopBody(MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI, Register OpSrcReg, Register OpDestReg, Register ElementCountReg, Register TotalIterationsReg, bool IsMemcpy)
Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and t2DoLoopEnd.
static SDValue PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformBUILD_VECTORCombine - Target-specific dag combine xforms for ISD::BUILD_VECTOR.
static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformMinMaxCombine - Target-specific DAG combining for creating truncating saturates.
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static Function * getFunction(Constant *C)
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
static Value * LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctpop of V before the specified instruction IP.
loop Loop Strength Reduction
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
This file describes how to lower LLVM code to machine code.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Assuming the flags are set by MI(a,b), return the condition code if we modify the instructions such t...
static constexpr int Concat[]
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
virtual const ARMBaseRegisterInfo & getRegisterInfo() const =0
const uint32_t * getSjLjDispatchPreservedMask(const MachineFunction &MF) const
const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const override
Code Generation virtual methods...
Register getFrameRegister(const MachineFunction &MF) const override
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
const uint32_t * getTLSCallPreservedMask(const MachineFunction &MF) const
const uint32_t * getThisReturnPreservedMask(const MachineFunction &MF, CallingConv::ID) const
getThisReturnPreservedMask - Returns a call preserved mask specific to the case that 'returned' is on...
static ARMConstantPoolConstant * Create(const Constant *C, unsigned ID)
static ARMConstantPoolMBB * Create(LLVMContext &C, const MachineBasicBlock *mbb, unsigned ID, unsigned char PCAdj)
static ARMConstantPoolSymbol * Create(LLVMContext &C, StringRef s, unsigned ID, unsigned char PCAdj)
ARMConstantPoolValue - ARM specific constantpool value.
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
int getVarArgsFrameIndex() const
int getPromotedConstpoolIncrease() const
SmallPtrSet< const GlobalVariable *, 2 > & getGlobalsPromotedToConstantPool()
void setArgumentStackToRestore(unsigned v)
bool branchTargetEnforcement() const
unsigned createPICLabelUId()
void setPromotedConstpoolIncrease(int Sz)
bool isThumb1OnlyFunction() const
void setArgRegsSaveSize(unsigned s)
bool shouldSignReturnAddress() const
bool isCmseNSEntryFunction() const
void setReturnRegsCount(unsigned s)
void setVarArgsFrameIndex(int Index)
unsigned getArgRegsSaveSize() const
void markGlobalAsPromotedToConstantPool(const GlobalVariable *GV)
Indicate to the backend that GV has had its storage changed to inside a constant pool.
void setIsSplitCSR(bool s)
void setArgumentStackSize(unsigned size)
unsigned getArgumentStackSize() const
bool isTargetMachO() const
bool isTargetAEABI() const
bool supportsTailCall() const
const Triple & getTargetTriple() const
const ARMBaseInstrInfo * getInstrInfo() const override
bool isThumb1Only() const
bool hasFPARMv8Base() const
bool isTargetWindows() const
bool isGVIndirectSymbol(const GlobalValue *GV) const
True if the GV will be accessed via an indirect symbol.
const ARMTargetLowering * getTargetLowering() const override
bool isTargetDarwin() const
const ARMBaseRegisterInfo * getRegisterInfo() const override
bool isTargetAndroid() const
bool isTargetCOFF() const
bool isTargetGNUAEABI() const
bool isTargetWatchOS() const
bool useNEONForSinglePrecisionFP() const
const InstrItineraryData * getInstrItineraryData() const override
getInstrItins - Return the instruction itineraries based on subtarget selection.
bool isTargetWatchABI() const
bool hasAnyDataBarrier() const
bool isTargetDriverKit() const
bool allowsUnalignedMem() const
bool isTargetMuslAEABI() const
bool isTargetLinux() const
unsigned getPrefLoopLogAlignment() const
bool isTargetHardFloat() const
Align getDualLoadStoreAlignment() const
bool isReadOnly(const GlobalValue *GV) const
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
bool shouldInsertFencesForAtomic(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const override
Return the correct alignment for the current calling convention.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
isLegalAddressingMode - Return true if the addressing mode represented by AM is legal for this target...
const ARMSubtarget * getSubtarget() const
bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const
bool isLegalT1ScaledAddressingMode(const AddrMode &AM, EVT VT) const
Returns true if the addressing mode representing by AM is legal for the Thumb1 target,...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPreIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mod...
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
ReplaceNodeResults - Replace the results of node with an illegal result type with new values built ou...
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
bool isLegalAddImmediate(int64_t Imm) const override
isLegalAddImmediate - Return true if the specified immediate is legal add immediate,...
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
SDValue PerformMVETruncCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const override
isFPImmLegal - Returns true if the target can instruction select the specified FP immediate natively.
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
SDValue PerformIntrinsicCombine(SDNode *N, DAGCombinerInfo &DCI) const
PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
bool isDesirableToCommuteXorWithShift(const SDNode *N) const override
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
bool ExpandInlineAsm(CallInst *CI) const override
This hook allows the target to expand an inline asm call to be explicit llvm code if it wants to.
SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const
PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const override
Create the IR node for the given complex deinterleaving operation.
bool isComplexDeinterleavingSupported() const override
Does this target support complex deinterleaving.
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
SDValue PerformMVEExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
getSetCCResultType - Return the value type to use for ISD::SETCC.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool isVarArg) const
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
createFastISel - This method returns a target specific FastISel object, or null if the target does no...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
isShuffleMaskLegal - Targets can use this to indicate that they only support some VECTOR_SHUFFLE oper...
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const override
getRegClassFor - Return the register class that should be used for the specified value type.
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vstN intrinsic.
ARMTargetLowering(const TargetMachine &TM, const ARMSubtarget &STI)
bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const override
Does this target support complex deinterleaving with the given operation and type.
SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const
PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const override
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
Instruction * makeDMB(IRBuilderBase &Builder, ARM_MB::MemBOpt Domain) const
bool isLegalICmpImmediate(int64_t Imm) const override
isLegalICmpImmediate - Return true if the specified immediate is legal icmp immediate,...
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool isVarArg) const
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
allowsMisalignedMemoryAccesses - Returns true if the target allows unaligned memory accesses of the s...
bool isLegalInterleavedAccessType(unsigned Factor, FixedVectorType *VecTy, Align Alignment, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
bool isVectorLoadExtDesirable(SDValue ExtVal) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const override
Return true if the target can combine store(extractelement VectorTy, Idx).
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vldN intrinsic.
bool useSoftFloat() const override
bool alignLoopsWithOptSize() const override
Should loops be aligned even when the function is marked OptSize (but not MinSize).
SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
LowerAsmOperandForConstraint - Lower the specified operand into the Ops vector.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const override
Returns true if an argument of type Ty needs to be passed in a contiguous block of registers in calli...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPostIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mo...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
bool isFloatingPointOperation() const
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
The address of a basic block.
static BranchProbability getZero()
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
CCState - This class holds information needed while lowering arguments and return values.
void getInRegsParamInfo(unsigned InRegsParamRecordIndex, unsigned &BeginReg, unsigned &EndReg) const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
void rewindByValRegsInfo()
unsigned getInRegsParamsProcessed() const
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void addInRegsParamInfo(unsigned RegBegin, unsigned RegEnd)
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
unsigned getInRegsParamsCount() const
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
unsigned getValNo() const
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
Value * getCalledOperand() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
const APFloat & getValueAPF() const
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
Align getStackAlignment() const
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPreferredAlign(const GlobalVariable *GV) const
Returns the preferred alignment of the specified global.
StringRef getPrivateGlobalPrefix() const
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for unsupported feature in backend.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
const GlobalValue * getGlobal() const
bool hasExternalWeakLinkage() const
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
bool isStrongDefinitionForLinker() const
Returns true if this global's definition will be the one chosen by the linker.
@ InternalLinkage
Rename collisions when linking (static functions).
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
TargetInstrInfo overrides.
Common base class shared among various IRBuilders.
Value * CreateZExtOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
BasicBlock * GetInsertBlock() const
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateTruncOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::optional< unsigned > getOperandCycle(unsigned ItinClassIndx, unsigned OperandIdx) const
Return the cycle for the given class and operand.
bool isEmpty() const
Returns true if there are no itineraries.
bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
const BasicBlock * getParent() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
static bool LowerToByteSwap(CallInst *CI)
Try to replace a call instruction with a call to a bswap intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Describe properties that are true of each instruction in the target description file.
unsigned getSchedClass() const
Return the scheduling class for this instruction.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
bool isEHPad() const
Returns true if the block is a landing pad.
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool canFallThrough()
Return true if the block can implicitly transfer control to the block after it by falling off the end...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
pred_iterator pred_begin()
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void moveAfter(MachineBasicBlock *NewBefore)
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void computeMaxCallFrameSize(MachineFunction &MF, std::vector< MachineBasicBlock::iterator > *FrameSDOps=nullptr)
Computes the maximum size of a callframe.
void setAdjustsStack(bool V)
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool hasVAStart() const
Returns true if the function calls the llvm.va_start intrinsic.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
int getFunctionContextIndex() const
Return the index for the function context object.
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & reset(Property P)
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i....
Representation of each machine instruction.
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
const MachineOperand & getOperand(unsigned i) const
unsigned createJumpTableIndex(const std::vector< MachineBasicBlock * > &DestBBs)
createJumpTableIndex - Create a new jump table.
@ EK_Inline
EK_Inline - Jump table entries are emitted inline at their point of use.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
void setIsDef(bool Val=true)
Change a def to a use, or a use to a def.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class is used to represent an MLOAD node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
virtual void print(raw_ostream &OS, const Module *M) const
print - Print out the internal state of the pass.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< use_iterator > uses()
size_t use_size() const
Return the number of uses of this node.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
void setFlags(SDNodeFlags NewFlags)
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
void addCallSiteInfo(const SDNode *Node, CallSiteInfo &&CallInfo)
Set CallSiteInfo to be associated with Node.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
StringRef - Represent a constant reference to a string, i.e.
const unsigned char * bytes_end() const
constexpr size_t size() const
size - Get the string size.
const unsigned char * bytes_begin() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
void setCmpLibcallCC(RTLIB::Libcall Call, ISD::CondCode CC)
Override the default CondCode to be used to test the result of the comparison libcall against zero.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
void setMinStackArgumentAlignment(Align Alignment)
Set the minimum stack alignment of an argument.
const TargetMachine & getTargetMachine() const
void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
virtual unsigned getMaxSupportedInterleaveFactor() const
Get the maximum supported factor for interleaved memory accesses.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
bool isPositionIndependent() const
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
bool isOSMSVCRT() const
Is this a "Windows" OS targeting a "MSVCRT.dll" environment.
bool isOSVersionLT(unsigned Major, unsigned Minor=0, unsigned Micro=0) const
Helper function for doing comparisons against version numbers included in the target triple.
bool isWindowsMSVCEnvironment() const
Checks if the environment could be MSVC.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isArrayTy() const
True if this is an instance of ArrayType.
bool isPointerTy() const
True if this is an instance of PointerType.
Type * getArrayElementType() const
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static CondCodes getOppositeCondition(CondCodes CC)
@ SECREL
Thread Pointer Offset.
@ SBREL
Section Relative (Windows TLS)
@ GOTTPOFF
Global Offset Table, PC Relative.
@ TPOFF
Global Offset Table, Thread Pointer Offset.
TOF
Target Operand Flag enum.
@ MO_NONLAZY
MO_NONLAZY - This is an independent flag, on a symbol operand "FOO" it represents a symbol which,...
@ MO_SBREL
MO_SBREL - On a symbol operand, this represents a static base relative relocation.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_GOT
MO_GOT - On a symbol operand, this represents a GOT relative relocation.
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
unsigned getAM2Offset(unsigned AM2Opc)
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned createVMOVModImm(unsigned OpCmode, unsigned Val)
int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
int getFP32FP16Imm(const APInt &Imm)
If this is a FP16Imm encoded as a fp32 value, return the 8-bit encoding for it.
AddrOpc getAM2Op(unsigned AM2Opc)
bool isBitFieldInvertedMask(unsigned v)
const unsigned FPStatusBits
const unsigned FPReservedBits
const unsigned RoundingBitsPos
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Swift
Calling convention for Swift.
@ ARM_APCS
ARM Procedure Calling Standard (obsolete, but still used on some targets).
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ ARM_AAPCS
ARM Architecture Procedure Calling Standard calling convention (aka EABI).
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool CC_ARM_APCS_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool RetCC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns true if Val1 has a lower Constant Materialization Cost than Val2.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool FastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool RetCC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
void shuffle(Iterator first, Iterator last, RNG &&g)
static std::array< MachineOperand, 2 > predOps(ARMCC::CondCodes Pred, unsigned PredReg=0)
Get the operands corresponding to the given Pred value.
bool CC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
bool CC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_Win32_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
ComplexDeinterleavingOperation
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
ComplexDeinterleavingRotation
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
static MachineOperand t1CondCodeOp(bool isDead=false)
Get the operand corresponding to the conditional code result for Thumb1.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
static MachineOperand condCodeOp(unsigned CCReg=0)
Get the operand corresponding to the conditional code result.
bool isVREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isVREVMask - Check if a vector shuffle corresponds to a VREV instruction with the specified blocksize...
unsigned gettBLXrOpcode(const MachineFunction &MF)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
unsigned convertAddSubFlagsOpcode(unsigned OldOpc)
Map pseudo instructions that imply an 'S' bit onto real opcodes.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
static const unsigned PerfectShuffleTable[6561+1]
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Load/store instruction that can be merged with a base address update.
SDNode * N
Instruction that updates a pointer.
unsigned ConstInc
Pointer increment value if it is a constant, or 0 otherwise.
SDValue Inc
Pointer increment operand.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
bool isUnknown() const
Returns true if we don't know any bits.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setInRegister(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)