84#include "llvm/IR/IntrinsicsARM.h"
119#define DEBUG_TYPE "arm-isel"
122STATISTIC(NumOptimizedImms,
"Number of times immediates were optimized");
123STATISTIC(NumMovwMovt,
"Number of GAs materialized with movw + movt");
124STATISTIC(NumLoopByVals,
"Number of loops generated for byval arguments");
126 "Number of constants with their storage promoted into constant pools");
130 cl::desc(
"Enable / disable ARM interworking (for debugging only)"),
135 cl::desc(
"Enable / disable promotion of unnamed_addr constants into "
140 cl::desc(
"Maximum size of constant to promote into a constant pool"),
144 cl::desc(
"Maximum size of ALL constants to promote into a constant pool"),
149 cl::desc(
"Maximum interleave factor for MVE VLDn to generate."),
154 cl::desc(
"Maximum number of base-updates to check generating postindex."),
162 ARM::R0, ARM::R1, ARM::R2, ARM::R3
176void ARMTargetLowering::addTypeForNEON(
MVT VT,
MVT PromotedLdStVT) {
177 if (VT != PromotedLdStVT) {
186 if (ElemTy != MVT::f64)
190 if (ElemTy == MVT::i32) {
234void ARMTargetLowering::addDRTypeForNEON(
MVT VT) {
236 addTypeForNEON(VT, MVT::f64);
239void ARMTargetLowering::addQRTypeForNEON(
MVT VT) {
241 addTypeForNEON(VT, MVT::v2f64);
244void ARMTargetLowering::setAllExpand(
MVT VT) {
257void ARMTargetLowering::addAllExtLoads(
const MVT From,
const MVT To,
264void ARMTargetLowering::addMVEVectorTypes(
bool HasMVEFP) {
265 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
267 for (
auto VT : IntTypes) {
342 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
343 for (
auto VT : FloatTypes) {
417 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
418 for (
auto VT : LongTypes) {
435 addAllExtLoads(MVT::v8i16, MVT::v8i8,
Legal);
436 addAllExtLoads(MVT::v4i32, MVT::v4i16,
Legal);
437 addAllExtLoads(MVT::v4i32, MVT::v4i8,
Legal);
454 for (
auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
463 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
464 for (
auto VT : pTypes) {
509 return ST.hasAnyDataBarrier() && (!ST.isThumb() || ST.hasV8MBaselineOps());
519 RegInfo(Subtarget->getRegisterInfo()),
520 Itins(Subtarget->getInstrItineraryData()) {
526 const Triple &TT = TM.getTargetTriple();
528 if (Subtarget->isThumb1Only())
533 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only() &&
534 Subtarget->hasFPRegs()) {
538 if (!Subtarget->hasVFP2Base()) {
539 setAllExpand(MVT::f32);
548 if (!Subtarget->hasFP64()) {
549 setAllExpand(MVT::f64);
559 if (Subtarget->hasFullFP16()) {
574 if (Subtarget->hasBF16()) {
576 setAllExpand(MVT::bf16);
577 if (!Subtarget->hasFullFP16()) {
596 addAllExtLoads(VT, InnerVT,
Expand);
605 if (!Subtarget->isThumb1Only() && !Subtarget->hasV8_1MMainlineOps())
608 if (!Subtarget->hasV8_1MMainlineOps())
611 if (!Subtarget->isThumb1Only())
620 if (Subtarget->hasMVEIntegerOps())
621 addMVEVectorTypes(Subtarget->hasMVEFloatOps());
624 if (Subtarget->hasLOB()) {
628 if (Subtarget->hasNEON()) {
629 addDRTypeForNEON(MVT::v2f32);
630 addDRTypeForNEON(MVT::v8i8);
631 addDRTypeForNEON(MVT::v4i16);
632 addDRTypeForNEON(MVT::v2i32);
633 addDRTypeForNEON(MVT::v1i64);
635 addQRTypeForNEON(MVT::v4f32);
636 addQRTypeForNEON(MVT::v2f64);
637 addQRTypeForNEON(MVT::v16i8);
638 addQRTypeForNEON(MVT::v8i16);
639 addQRTypeForNEON(MVT::v4i32);
640 addQRTypeForNEON(MVT::v2i64);
642 if (Subtarget->hasFullFP16()) {
643 addQRTypeForNEON(MVT::v8f16);
644 addDRTypeForNEON(MVT::v4f16);
647 if (Subtarget->hasBF16()) {
648 addQRTypeForNEON(MVT::v8bf16);
649 addDRTypeForNEON(MVT::v4bf16);
653 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
693 if (Subtarget->hasNEON()) {
806 if (!Subtarget->hasVFP4Base()) {
815 for (
MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
824 for (
auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
833 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
841 if (Subtarget->hasMVEIntegerOps()) {
846 if (Subtarget->hasMVEFloatOps()) {
850 if (!Subtarget->hasFP64()) {
902 if (!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) {
903 if (Subtarget->hasFullFP16()) {
909 if (Subtarget->hasFP16()) {
939 if (!Subtarget->isThumb1Only()) {
958 if (TT.isTargetAEABI() && !Subtarget->allowsUnalignedMem()) {
970 if (!Subtarget->isThumb1Only()) {
979 if (Subtarget->hasDSP()) {
989 if (Subtarget->hasBaseDSP()) {
997 if (Subtarget->isThumb1Only()) {
1001 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops()
1002 || (Subtarget->isThumb2() && !Subtarget->hasDSP()))
1017 if (Subtarget->hasMVEIntegerOps())
1021 if (Subtarget->isThumb1Only()) {
1027 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops())
1041 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) {
1050 if (Subtarget->hasPerfMon())
1054 if (!Subtarget->hasV6Ops())
1057 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1058 : Subtarget->hasDivideInARMMode();
1065 if (TT.isOSWindows() && !Subtarget->hasDivideInThumbMode()) {
1077 if (TT.isTargetAEABI() || TT.isAndroid() || TT.isTargetGNUAEABI() ||
1078 TT.isTargetMuslAEABI() || TT.isOSFuchsia() || TT.isOSWindows()) {
1081 HasStandaloneRem =
false;
1108 if (TT.isOSWindows())
1115 InsertFencesForAtomic =
false;
1120 if (!Subtarget->isThumb() || !Subtarget->isMClass())
1125 if (!Subtarget->hasAcquireRelease() ||
1128 InsertFencesForAtomic =
true;
1134 if (Subtarget->hasDataBarrier())
1135 InsertFencesForAtomic =
true;
1146 {MVT::i8, MVT::i16, MVT::i32, MVT::i64},
LibCall);
1152 if (TT.isOSLinux() || (!Subtarget->isMClass() && Subtarget->hasV6Ops())) {
1164 }
else if ((Subtarget->isMClass() && Subtarget->hasV8MBaselineOps()) ||
1165 Subtarget->hasForced32BitAtomics()) {
1179 if (!Subtarget->hasV6Ops()) {
1185 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1186 !Subtarget->isThumb1Only()) {
1215 if (Subtarget->hasFullFP16()) {
1225 if (Subtarget->hasFullFP16())
1240 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2Base() &&
1241 !Subtarget->isThumb1Only()) {
1248 if (!Subtarget->hasVFP4Base()) {
1254 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) {
1256 if (!Subtarget->hasFPARMv8Base() || !Subtarget->hasFP64()) {
1264 if (!Subtarget->hasFP16()) {
1281 if (Subtarget->hasFPARMv8Base()) {
1291 if (Subtarget->hasFP64())
1295 if (Subtarget->hasNEON()) {
1305 if (Subtarget->hasFullFP16()) {
1342 if (Subtarget->hasNEON()) {
1354 if (Subtarget->hasV8Ops()) {
1364 if (Subtarget->hasFullFP16()) {
1387 if (TT.isOSWindows()) {
1404 if (Subtarget->hasMVEIntegerOps())
1407 if (Subtarget->hasV6Ops())
1409 if (Subtarget->isThumb1Only())
1412 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1413 Subtarget->isThumb2()) {
1419 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
1420 !Subtarget->hasVFP2Base() || Subtarget->hasMinSize())
1451 return Subtarget->useSoftFloat();
1455 return !Subtarget->isThumb1Only() && VT.
getSizeInBits() <= 32;
1468std::pair<const TargetRegisterClass *, uint8_t>
1479 case MVT::f32:
case MVT::f64:
case MVT::v8i8:
case MVT::v4i16:
1480 case MVT::v2i32:
case MVT::v1i64:
case MVT::v2f32:
1481 RRC = &ARM::DPRRegClass;
1486 if (Subtarget->useNEONForSinglePrecisionFP())
1489 case MVT::v16i8:
case MVT::v8i16:
case MVT::v4i32:
case MVT::v2i64:
1490 case MVT::v4f32:
case MVT::v2f64:
1491 RRC = &ARM::DPRRegClass;
1495 RRC = &ARM::DPRRegClass;
1499 RRC = &ARM::DPRRegClass;
1503 return std::make_pair(RRC,
Cost);
1512 if (Subtarget->hasMVEIntegerOps())
1527 if (Subtarget->hasNEON()) {
1528 if (VT == MVT::v4i64)
1529 return &ARM::QQPRRegClass;
1530 if (VT == MVT::v8i64)
1531 return &ARM::QQQQPRRegClass;
1533 if (Subtarget->hasMVEIntegerOps()) {
1534 if (VT == MVT::v4i64)
1535 return &ARM::MQQPRRegClass;
1536 if (VT == MVT::v8i64)
1537 return &ARM::MQQQQPRRegClass;
1546 Align &PrefAlign)
const {
1553 (Subtarget->hasV6Ops() && !Subtarget->isMClass() ?
Align(8) :
Align(4));
1565 unsigned NumVals =
N->getNumValues();
1569 for (
unsigned i = 0; i != NumVals; ++i) {
1570 EVT VT =
N->getValueType(i);
1571 if (VT == MVT::Glue || VT == MVT::Other)
1577 if (!
N->isMachineOpcode())
1585 if (
MCID.getNumDefs() == 0)
1587 if (!Itins->isEmpty() &&
1588 Itins->getOperandCycle(
MCID.getSchedClass(), 0) > 2U)
1602 return Const->getZExtValue() == 16;
1610 return Const->getZExtValue() == 16;
1618 return Const->getZExtValue() == 16;
1687 bool isVarArg)
const {
1706 if (!Subtarget->isAAPCS_ABI())
1708 else if (Subtarget->isTargetHardFloat() && !isVarArg)
1714 if (!Subtarget->isAAPCS_ABI()) {
1715 if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() && !isVarArg)
1718 }
else if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() &&
1719 !Subtarget->useSoftFloat() && !isVarArg)
1727 bool isVarArg)
const {
1728 return CCAssignFnForNode(
CC,
false, isVarArg);
1732 bool isVarArg)
const {
1733 return CCAssignFnForNode(
CC,
true, isVarArg);
1740 bool isVarArg)
const {
1767 if (Subtarget->hasFullFP16()) {
1768 Val = DAG.
getNode(ARMISD::VMOVhr, dl, ValVT, Val);
1780 if (Subtarget->hasFullFP16()) {
1781 Val = DAG.
getNode(ARMISD::VMOVrh, dl,
1794SDValue ARMTargetLowering::LowerCallResult(
1798 SDValue ThisVal,
bool isCmseNSCall)
const {
1806 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
1807 CCValAssign VA = RVLocs[i];
1811 if (i == 0 && isThisReturn) {
1813 "unexpected return calling convention register assignment");
1831 if (!Subtarget->isLittle())
1833 Val = DAG.
getNode(ARMISD::VMOVDRR, dl, MVT::f64,
Lo,
Hi);
1848 if (!Subtarget->isLittle())
1850 Val = DAG.
getNode(ARMISD::VMOVDRR, dl, MVT::f64,
Lo,
Hi);
1880 const ISD::InputArg &Arg = Ins[VA.
getValNo()];
1891std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
1893 bool IsTailCall,
int SPDiff)
const {
1895 MachinePointerInfo DstInfo;
1915 return std::make_pair(DstAddr, DstInfo);
1924ARMTargetLowering::ByValCopyKind ARMTargetLowering::ByValNeedsCopyForTailCall(
1937 if (!SrcFrameIdxNode || !DstFrameIdxNode)
1940 int SrcFI = SrcFrameIdxNode->getIndex();
1941 int DstFI = DstFrameIdxNode->getIndex();
1943 "byval passed in non-fixed stack slot");
1965 if (SrcOffset == DstOffset)
1973 RegsToPassVector &RegsToPass,
1980 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
1981 unsigned id = Subtarget->isLittle() ? 0 : 1;
1993 MachinePointerInfo DstInfo;
1994 std::tie(DstAddr, DstInfo) =
1995 computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
2012 SelectionDAG &DAG = CLI.
DAG;
2014 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.
Outs;
2015 SmallVectorImpl<SDValue> &OutVals = CLI.
OutVals;
2016 SmallVectorImpl<ISD::InputArg> &Ins = CLI.
Ins;
2023 const CallBase *CB = CLI.
CB;
2026 ARMFunctionInfo *AFI = MF.
getInfo<ARMFunctionInfo>();
2028 MachineFunction::CallSiteInfo CSInfo;
2029 bool isStructRet = (Outs.
empty()) ?
false : Outs[0].Flags.isSRet();
2030 bool isThisReturn =
false;
2031 bool isCmseNSCall =
false;
2032 bool isSibCall =
false;
2033 bool PreferIndirect =
false;
2034 bool GuardWithBTI =
false;
2044 !Subtarget->noBTIAtReturnTwice())
2052 isCmseNSCall =
true;
2055 if (!Subtarget->supportsTailCall())
2071 PreferIndirect = Subtarget->isThumb() && Subtarget->hasMinSize() &&
2072 count_if(GV->users(), [&BB](
const User *U) {
2073 return isa<Instruction>(U) &&
2074 cast<Instruction>(U)->getParent() == BB;
2081 IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs, PreferIndirect);
2095 "site marked musttail");
2098 unsigned NumBytes = CCInfo.getStackSize();
2107 if (isTailCall && !isSibCall) {
2108 auto FuncInfo = MF.
getInfo<ARMFunctionInfo>();
2109 unsigned NumReusableBytes = FuncInfo->getArgumentStackSize();
2114 assert(StackAlign &&
"data layout string is missing stack alignment");
2115 NumBytes =
alignTo(NumBytes, *StackAlign);
2120 SPDiff = NumReusableBytes - NumBytes;
2124 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (
unsigned)-SPDiff)
2140 RegsToPassVector RegsToPass;
2148 for (
const ForwardedRegister &
F : Forwards) {
2150 RegsToPass.push_back(std::make_pair(
F.PReg, Val));
2160 DenseMap<unsigned, SDValue> ByValTemporaries;
2164 for (
const CCValAssign &VA : ArgLocs) {
2166 SDValue Src = OutVals[ArgIdx];
2167 ISD::ArgFlagsTy
Flags = Outs[ArgIdx].Flags;
2169 if (!
Flags.isByVal())
2173 MachinePointerInfo DstInfo;
2174 std::tie(Dst, DstInfo) =
2175 computeAddrForCallArg(dl, DAG, VA,
SDValue(),
true, SPDiff);
2176 ByValCopyKind
Copy = ByValNeedsCopyForTailCall(DAG, Src, Dst, Flags);
2178 if (Copy == NoCopy) {
2183 }
else if (Copy == CopyOnce) {
2187 ByValTemporaries[ArgIdx] = Src;
2189 assert(Copy == CopyViaTemp &&
"unexpected enum value");
2193 int TempFrameIdx = MFI.CreateStackObject(
2194 Flags.getByValSize(),
Flags.getNonZeroByValAlign(),
false);
2202 SDVTList VTs = DAG.
getVTList(MVT::Other, MVT::Glue);
2203 SDValue Ops[] = {Chain, Temp, Src, SizeNode, AlignNode};
2205 DAG.
getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs,
Ops));
2206 ByValTemporaries[ArgIdx] = Temp;
2209 if (!ByValCopyChains.
empty())
2219 bool AfterFormalArgLoads =
false;
2223 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
2225 ++i, ++realArgIdx) {
2226 CCValAssign &VA = ArgLocs[i];
2227 SDValue Arg = OutVals[realArgIdx];
2228 ISD::ArgFlagsTy
Flags = Outs[realArgIdx].Flags;
2229 bool isByVal =
Flags.isByVal();
2249 if (isTailCall && VA.
isMemLoc() && !AfterFormalArgLoads) {
2251 if (ByValTempChain) {
2256 for (
unsigned I = 0;
I < OutVals.
size(); ++
I) {
2257 if (Outs[
I].
Flags.isByVal())
2265 FrameIndexSDNode *FIN =
2270 if (!MFI.isFixedObjectIndex(FIN->
getIndex()))
2273 for (
const CCValAssign &VA : ArgLocs) {
2281 if (!IncomingLoad.
empty()) {
2289 AfterFormalArgLoads =
true;
2301 auto ArgVT = Outs[realArgIdx].ArgVT;
2302 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2320 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2321 StackPtr, MemOpChains, isTailCall, SPDiff);
2325 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2326 StackPtr, MemOpChains, isTailCall, SPDiff);
2330 MachinePointerInfo DstInfo;
2331 std::tie(DstAddr, DstInfo) =
2332 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2336 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2337 StackPtr, MemOpChains, isTailCall, SPDiff);
2339 if (realArgIdx == 0 &&
Flags.isReturned() && !
Flags.isSwiftSelf() &&
2340 Outs[0].VT == MVT::i32) {
2342 "unexpected calling convention register assignment");
2344 "unexpected use of 'returned'");
2345 isThisReturn =
true;
2350 RegsToPass.push_back(std::make_pair(VA.
getLocReg(), Arg));
2351 }
else if (isByVal) {
2353 unsigned offset = 0;
2357 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2358 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2361 bool NeedsStackCopy;
2362 if (
auto It = ByValTemporaries.
find(realArgIdx);
2363 It != ByValTemporaries.
end()) {
2364 ByValSrc = It->second;
2365 NeedsStackCopy =
true;
2368 NeedsStackCopy = !isTailCall;
2372 if (CurByValIdx < ByValArgsCount) {
2373 unsigned RegBegin, RegEnd;
2374 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
2378 for (i = 0, j = RegBegin;
j < RegEnd; i++,
j++) {
2382 DAG.
getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo(),
2385 RegsToPass.push_back(std::make_pair(j,
Load));
2390 offset = RegEnd - RegBegin;
2392 CCInfo.nextInRegsParam();
2397 if (NeedsStackCopy &&
Flags.getByValSize() > 4 * offset) {
2400 MachinePointerInfo DstInfo;
2401 std::tie(Dst, DstInfo) =
2402 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2410 SDVTList VTs = DAG.
getVTList(MVT::Other, MVT::Glue);
2411 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2418 MachinePointerInfo DstInfo;
2419 std::tie(DstAddr, DstInfo) =
2420 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2427 if (!MemOpChains.
empty())
2433 for (
const auto &[
Reg,
N] : RegsToPass) {
2441 bool isDirect =
false;
2444 const Triple &
TT = TM.getTargetTriple();
2445 const GlobalValue *GVal =
nullptr;
2447 GVal =
G->getGlobal();
2448 bool isStub = !TM.shouldAssumeDSOLocal(GVal) &&
TT.isOSBinFormatMachO();
2450 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass());
2451 bool isLocalARMFunc =
false;
2454 if (Subtarget->genLongCalls()) {
2456 if (isPIC && Subtarget->genExecuteOnly())
2458 "position-independent code is not supported");
2459 if (Subtarget->isROPI())
2466 if (Subtarget->genExecuteOnly()) {
2469 if (Subtarget->useMovt())
2492 Addr = DAG.
getNode(ARMISD::Wrapper, dl, MVT::i32, Addr);
2498 const char *Sym = S->getSymbol();
2500 if (Subtarget->genExecuteOnly()) {
2503 if (Subtarget->useMovt())
2510 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
2516 CPAddr = DAG.
getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
2521 Callee = DAG.
getNode(ARMISD::PIC_ADD, dl, PtrVt, GOTOffset, PICLabel);
2533 Addr = DAG.
getNode(ARMISD::Wrapper, dl, MVT::i32, Addr);
2540 if (!PreferIndirect) {
2545 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !
ARMInterworking);
2547 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2548 assert(
TT.isOSBinFormatMachO() &&
"WrapperPIC use on non-MachO?");
2550 ARMISD::WrapperPIC, dl, PtrVt,
2557 }
else if (Subtarget->isTargetCOFF()) {
2558 assert(Subtarget->isTargetWindows() &&
2559 "Windows is the only supported COFF target");
2563 else if (!TM.shouldAssumeDSOLocal(GVal))
2570 DAG.
getNode(ARMISD::Wrapper, dl, PtrVt, Callee),
2579 const char *Sym = S->getSymbol();
2580 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2582 ARMConstantPoolValue *CPV =
2584 ARMPCLabelIndex, 4);
2586 CPAddr = DAG.
getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
2591 Callee = DAG.
getNode(ARMISD::PIC_ADD, dl, PtrVt, Callee, PICLabel);
2598 assert(!isARMFunc && !isDirect &&
2599 "Cannot handle call to ARM function or direct call");
2603 "call to non-secure function would require "
2604 "passing arguments on stack",
2610 "call to non-secure function would return value through pointer",
2617 if (Subtarget->isThumb()) {
2619 CallOpc = ARMISD::t2CALL_BTI;
2620 else if (isCmseNSCall)
2621 CallOpc = ARMISD::tSECALL;
2622 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2623 CallOpc = ARMISD::CALL_NOLINK;
2625 CallOpc = ARMISD::CALL;
2627 if (!isDirect && !Subtarget->hasV5TOps())
2628 CallOpc = ARMISD::CALL_NOLINK;
2629 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2631 !Subtarget->hasMinSize())
2633 CallOpc = ARMISD::CALL_NOLINK;
2635 CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL;
2642 if (isTailCall && !isSibCall) {
2647 std::vector<SDValue>
Ops;
2648 Ops.push_back(Chain);
2649 Ops.push_back(Callee);
2657 for (
const auto &[
Reg,
N] : RegsToPass)
2661 const uint32_t *
Mask;
2662 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
2670 isThisReturn =
false;
2676 assert(Mask &&
"Missing call preserved mask for calling convention");
2680 Ops.push_back(InGlue);
2693 Chain = DAG.
getNode(CallOpc, dl, {MVT::Other, MVT::Glue},
Ops);
2707 Chain = DAG.
getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
2713 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2714 InVals, isThisReturn,
2715 isThisReturn ? OutVals[0] :
SDValue(), isCmseNSCall);
2722void ARMTargetLowering::HandleByVal(
CCState *State,
unsigned &
Size,
2723 Align Alignment)
const {
2731 unsigned AlignInRegs =
Alignment.value() / 4;
2732 unsigned Waste = (ARM::R4 -
Reg) % AlignInRegs;
2733 for (
unsigned i = 0; i < Waste; ++i)
2739 unsigned Excess = 4 * (ARM::R4 -
Reg);
2746 if (NSAAOffset != 0 &&
Size > Excess) {
2758 unsigned ByValRegBegin =
Reg;
2759 unsigned ByValRegEnd = std::min<unsigned>(
Reg +
Size / 4, ARM::R4);
2763 for (
unsigned i =
Reg + 1; i != ByValRegEnd; ++i)
2769 Size = std::max<int>(
Size - Excess, 0);
2777bool ARMTargetLowering::IsEligibleForTailCallOptimization(
2783 const SmallVectorImpl<ISD::OutputArg> &Outs = CLI.
Outs;
2784 const SmallVectorImpl<SDValue> &OutVals = CLI.
OutVals;
2785 const SmallVectorImpl<ISD::InputArg> &Ins = CLI.
Ins;
2786 const SelectionDAG &DAG = CLI.
DAG;
2791 assert(Subtarget->supportsTailCall());
2804 SmallSet<MCPhysReg, 5> AddressRegisters = {ARM::R0, ARM::R1, ARM::R2,
2806 if (!(Subtarget->isThumb1Only() ||
2807 MF.
getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(
true)))
2808 AddressRegisters.
insert(ARM::R12);
2809 for (
const CCValAssign &AL : ArgLocs)
2811 AddressRegisters.
erase(
AL.getLocReg());
2812 if (AddressRegisters.
empty()) {
2813 LLVM_DEBUG(
dbgs() <<
"false (no reg to hold function pointer)\n");
2832 <<
" (guaranteed tail-call CC)\n");
2833 return CalleeCC == CallerCC;
2838 bool isCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
2840 if (isCalleeStructRet != isCallerStructRet) {
2853 const GlobalValue *GV =
G->getGlobal();
2856 (!
TT.isOSWindows() ||
TT.isOSBinFormatELF() ||
2857 TT.isOSBinFormatMachO())) {
2874 const ARMBaseRegisterInfo *
TRI = Subtarget->getRegisterInfo();
2875 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
2876 if (CalleeCC != CallerCC) {
2877 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
2878 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) {
2887 const ARMFunctionInfo *AFI_Caller = MF.
getInfo<ARMFunctionInfo>();
2897 LLVM_DEBUG(
dbgs() <<
"false (parameters in CSRs do not match)\n");
2916 CCState CCInfo(CallConv, isVarArg, MF, RVLocs,
Context);
2925 StringRef IntKind =
F.getFnAttribute(
"interrupt").getValueAsString();
2938 if (IntKind ==
"" || IntKind ==
"IRQ" || IntKind ==
"FIQ" ||
2941 else if (IntKind ==
"SWI" || IntKind ==
"UNDEF")
2945 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
2950 return DAG.
getNode(ARMISD::INTRET_GLUE,
DL, MVT::Other, RetOps);
2972 bool isLittleEndian = Subtarget->isLittle();
2975 ARMFunctionInfo *AFI = MF.
getInfo<ARMFunctionInfo>();
2984 "secure entry function would return value through pointer",
2989 for (
unsigned i = 0, realRVLocIdx = 0;
2991 ++i, ++realRVLocIdx) {
2992 CCValAssign &VA = RVLocs[i];
2995 SDValue Arg = OutVals[realRVLocIdx];
2996 bool ReturnF16 =
false;
2998 if (Subtarget->hasFullFP16() && Subtarget->isTargetHardFloat()) {
3031 auto RetVT = Outs[realRVLocIdx].ArgVT;
3053 DAG.
getVTList(MVT::i32, MVT::i32), Half);
3057 HalfGPRs.
getValue(isLittleEndian ? 0 : 1), Glue);
3063 HalfGPRs.
getValue(isLittleEndian ? 1 : 0), Glue);
3075 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
3077 fmrrd.
getValue(isLittleEndian ? 0 : 1), Glue);
3082 fmrrd.
getValue(isLittleEndian ? 1 : 0), Glue);
3092 const ARMBaseRegisterInfo *
TRI = Subtarget->getRegisterInfo();
3118 !Subtarget->isMClass()) {
3119 if (Subtarget->isThumb1Only())
3126 return DAG.
getNode(RetNode, dl, MVT::Other, RetOps);
3129bool ARMTargetLowering::isUsedByReturnOnly(
SDNode *
N,
SDValue &Chain)
const {
3130 if (
N->getNumValues() != 1)
3132 if (!
N->hasNUsesOfValue(1, 0))
3136 SDNode *
Copy = *
N->user_begin();
3140 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3142 TCChain =
Copy->getOperand(0);
3143 }
else if (
Copy->getOpcode() == ARMISD::VMOVRRD) {
3144 SDNode *VMov =
Copy;
3146 SmallPtrSet<SDNode*, 2>
Copies;
3147 for (SDNode *U : VMov->
users()) {
3155 for (SDNode *U : VMov->
users()) {
3156 SDValue UseChain =
U->getOperand(0);
3164 if (
U->getOperand(
U->getNumOperands() - 1).getValueType() == MVT::Glue)
3172 if (!
Copy->hasOneUse())
3179 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3181 TCChain =
Copy->getOperand(0);
3186 bool HasRet =
false;
3187 for (
const SDNode *U :
Copy->users()) {
3188 if (
U->getOpcode() != ARMISD::RET_GLUE &&
3189 U->getOpcode() != ARMISD::INTRET_GLUE)
3201bool ARMTargetLowering::mayBeEmittedAsTailCall(
const CallInst *CI)
const {
3202 if (!Subtarget->supportsTailCall())
3219 &&
"LowerWRITE_REGISTER called for non-i64 type argument.");
3235 EVT PtrVT =
Op.getValueType();
3245 if (Subtarget->genExecuteOnly()) {
3250 auto GV =
new GlobalVariable(
3256 return LowerGlobalAddress(GA, DAG);
3262 if (Subtarget->isThumb1Only())
3263 CPAlign = std::max(CPAlign,
Align(4));
3269 return DAG.
getNode(ARMISD::Wrapper, dl, MVT::i32, Res);
3276 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3285 unsigned ARMPCLabelIndex = 0;
3291 if (!IsPositionIndependent) {
3294 unsigned PCAdj =
Subtarget->isThumb() ? 4 : 8;
3301 CPAddr = DAG.
getNode(ARMISD::Wrapper,
DL, PtrVT, CPAddr);
3305 if (!IsPositionIndependent)
3308 return DAG.
getNode(ARMISD::PIC_ADD,
DL, PtrVT, Result, PICLabel);
3336ARMTargetLowering::LowerGlobalTLSAddressDarwin(
SDValue Op,
3339 "This function expects a Darwin target");
3344 SDValue DescAddr = LowerGlobalAddressDarwin(
Op, DAG);
3350 MVT::i32,
DL, Chain, DescAddr,
3365 auto ARI =
static_cast<const ARMRegisterInfo *
>(
TRI);
3374 Chain, FuncTLVGet, DAG.
getRegister(ARM::R0, MVT::i32),
3380ARMTargetLowering::LowerGlobalTLSAddressWindows(
SDValue Op,
3383 "Windows specific TLS lowering");
3407 TLSArray = DAG.
getLoad(PtrVT,
DL, Chain, TLSArray, MachinePointerInfo());
3415 TLSIndex = DAG.
getNode(ARMISD::Wrapper,
DL, PtrVT, TLSIndex);
3416 TLSIndex = DAG.
getLoad(PtrVT,
DL, Chain, TLSIndex, MachinePointerInfo());
3422 MachinePointerInfo());
3429 DAG.
getNode(ARMISD::Wrapper,
DL, MVT::i32,
3442 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3444 ARMFunctionInfo *AFI = MF.
getInfo<ARMFunctionInfo>();
3446 ARMConstantPoolValue *CPV =
3457 Argument = DAG.
getNode(ARMISD::PIC_ADD, dl, PtrVT, Argument, PICLabel);
3464 TargetLowering::CallLoweringInfo CLI(DAG);
3469 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
3470 return CallResult.first;
3479 const GlobalValue *GV = GA->
getGlobal();
3485 SDValue ThreadPointer = DAG.
getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
3489 ARMFunctionInfo *AFI = MF.
getInfo<ARMFunctionInfo>();
3492 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3493 ARMConstantPoolValue *CPV =
3500 PtrVT, dl, Chain,
Offset,
3508 PtrVT, dl, Chain,
Offset,
3513 ARMConstantPoolValue *CPV =
3518 PtrVT, dl, Chain,
Offset,
3534 if (
TT.isOSDarwin())
3535 return LowerGlobalTLSAddressDarwin(
Op, DAG);
3537 if (
TT.isOSWindows())
3538 return LowerGlobalTLSAddressWindows(
Op, DAG);
3541 assert(
TT.isOSBinFormatELF() &&
"Only ELF implemented here");
3547 return LowerToTLSGeneralDynamicModel(GA, DAG);
3550 return LowerToTLSExecModels(GA, DAG, model);
3559 while (!Worklist.
empty()) {
3567 if (!
I ||
I->getParent()->getParent() !=
F)
3596 if (!GVar || !GVar->hasInitializer() ||
3597 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3598 !GVar->hasLocalLinkage())
3603 auto *
Init = GVar->getInitializer();
3604 if ((
TLI->isPositionIndependent() ||
TLI->getSubtarget()->isROPI()) &&
3605 Init->needsDynamicRelocation())
3617 unsigned RequiredPadding = 4 - (
Size % 4);
3618 bool PaddingPossible =
3619 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3624 unsigned PaddedSize =
Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3648 if (RequiredPadding != 4) {
3653 while (RequiredPadding--)
3665 ++NumConstpoolPromoted;
3666 return DAG.
getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3671 if (!(GV = GA->getAliaseeObject()))
3674 return V->isConstant();
3680 switch (
Subtarget->getTargetTriple().getObjectFormat()) {
3683 return LowerGlobalAddressWindows(
Op, DAG);
3685 return LowerGlobalAddressELF(
Op, DAG);
3687 return LowerGlobalAddressDarwin(
Op, DAG);
3699 if (GV->
isDSOLocal() && !Subtarget->genExecuteOnly())
3712 }
else if (Subtarget->isROPI() && IsRO) {
3717 }
else if (Subtarget->isRWPI() && !IsRO) {
3720 if (Subtarget->useMovt()) {
3723 RelAddr = DAG.
getNode(ARMISD::Wrapper, dl, PtrVT,
G);
3725 ARMConstantPoolValue *CPV =
3728 CPAddr = DAG.
getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3742 if (Subtarget->useMovt() || Subtarget->genExecuteOnly()) {
3743 if (Subtarget->useMovt())
3747 return DAG.
getNode(ARMISD::Wrapper, dl, PtrVT,
3751 CPAddr = DAG.
getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3760 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3761 "ROPI/RWPI not currently supported for Darwin");
3766 if (Subtarget->useMovt())
3777 if (Subtarget->isGVIndirectSymbol(GV))
3786 "non-Windows COFF is not supported");
3787 assert(Subtarget->useMovt() &&
3788 "Windows on ARM expects to use movw/movt");
3789 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3790 "ROPI/RWPI not currently supported for Windows");
3797 else if (!TM.shouldAssumeDSOLocal(GV))
3820 return DAG.
getNode(ARMISD::EH_SJLJ_SETJMP, dl,
3821 DAG.
getVTList(MVT::i32, MVT::Other),
Op.getOperand(0),
3822 Op.getOperand(1), Val);
3828 return DAG.
getNode(ARMISD::EH_SJLJ_LONGJMP, dl, MVT::Other,
Op.getOperand(0),
3835 return DAG.
getNode(ARMISD::EH_SJLJ_SETUP_DISPATCH, dl, MVT::Other,
3839SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
3842 Op.getConstantOperandVal(
Op.getOperand(0).getValueType() == MVT::Other);
3846 case Intrinsic::arm_gnu_eabi_mcount: {
3852 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
3853 const uint32_t *
Mask =
3855 assert(Mask &&
"Missing call preserved mask for calling convention");
3860 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
3864 if (Subtarget->isThumb())
3867 ARM::tBL_PUSHLR, dl, ResultTys,
3868 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
3869 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
3873 {ReturnAddress, Callee, RegisterMask, Chain}),
3882 unsigned IntNo =
Op.getConstantOperandVal(0);
3886 case Intrinsic::localaddress: {
3888 const auto *RegInfo = Subtarget->getRegisterInfo();
3889 unsigned Reg = RegInfo->getLocalAddressRegister(MF);
3891 Op.getSimpleValueType());
3893 case Intrinsic::eh_recoverfp: {
3899 "llvm.eh.recoverfp must take a function as the first argument");
3900 const auto *RegInfo = Subtarget->getRegisterInfo();
3903 MachineBasicBlock &
MBB = *MF.
begin();
3909 case Intrinsic::thread_pointer: {
3911 return DAG.
getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
3913 case Intrinsic::arm_cls: {
3917 const SDValue &Operand =
Op.getOperand(1);
3918 const EVT VTy =
Op.getValueType();
3921 case Intrinsic::arm_cls64: {
3927 case Intrinsic::arm_neon_vcls:
3928 case Intrinsic::arm_mve_vcls: {
3931 const EVT VTy =
Op.getValueType();
3934 case Intrinsic::eh_sjlj_lsda: {
3936 ARMFunctionInfo *AFI = MF.
getInfo<ARMFunctionInfo>();
3941 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
3942 ARMConstantPoolValue *CPV =
3946 CPAddr = DAG.
getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3951 if (IsPositionIndependent) {
3953 Result = DAG.
getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
3957 case Intrinsic::arm_neon_vabs:
3960 case Intrinsic::arm_neon_vabds:
3961 if (
Op.getValueType().isInteger())
3963 Op.getOperand(1),
Op.getOperand(2));
3965 case Intrinsic::arm_neon_vabdu:
3967 Op.getOperand(1),
Op.getOperand(2));
3968 case Intrinsic::arm_neon_vmulls:
3969 case Intrinsic::arm_neon_vmullu: {
3970 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
3971 ? ARMISD::VMULLs : ARMISD::VMULLu;
3972 return DAG.
getNode(NewOpc, SDLoc(
Op),
Op.getValueType(),
3973 Op.getOperand(1),
Op.getOperand(2));
3975 case Intrinsic::arm_neon_vminnm:
3976 case Intrinsic::arm_neon_vmaxnm: {
3977 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
3979 return DAG.
getNode(NewOpc, SDLoc(
Op),
Op.getValueType(),
3980 Op.getOperand(1),
Op.getOperand(2));
3982 case Intrinsic::arm_neon_vminu:
3983 case Intrinsic::arm_neon_vmaxu: {
3984 if (
Op.getValueType().isFloatingPoint())
3986 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
3988 return DAG.
getNode(NewOpc, SDLoc(
Op),
Op.getValueType(),
3989 Op.getOperand(1),
Op.getOperand(2));
3991 case Intrinsic::arm_neon_vmins:
3992 case Intrinsic::arm_neon_vmaxs: {
3994 if (!
Op.getValueType().isFloatingPoint()) {
3995 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
3997 return DAG.
getNode(NewOpc, SDLoc(
Op),
Op.getValueType(),
3998 Op.getOperand(1),
Op.getOperand(2));
4000 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4002 return DAG.
getNode(NewOpc, SDLoc(
Op),
Op.getValueType(),
4003 Op.getOperand(1),
Op.getOperand(2));
4005 case Intrinsic::arm_neon_vtbl1:
4006 return DAG.
getNode(ARMISD::VTBL1, SDLoc(
Op),
Op.getValueType(),
4007 Op.getOperand(1),
Op.getOperand(2));
4008 case Intrinsic::arm_neon_vtbl2:
4009 return DAG.
getNode(ARMISD::VTBL2, SDLoc(
Op),
Op.getValueType(),
4010 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4011 case Intrinsic::arm_mve_pred_i2v:
4012 case Intrinsic::arm_mve_pred_v2i:
4013 return DAG.
getNode(ARMISD::PREDICATE_CAST, SDLoc(
Op),
Op.getValueType(),
4015 case Intrinsic::arm_mve_vreinterpretq:
4016 return DAG.
getNode(ARMISD::VECTOR_REG_CAST, SDLoc(
Op),
Op.getValueType(),
4018 case Intrinsic::arm_mve_lsll:
4019 return DAG.
getNode(ARMISD::LSLL, SDLoc(
Op),
Op->getVTList(),
4020 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4021 case Intrinsic::arm_mve_asrl:
4022 return DAG.
getNode(ARMISD::ASRL, SDLoc(
Op),
Op->getVTList(),
4023 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4024 case Intrinsic::arm_mve_vsli:
4025 return DAG.
getNode(ARMISD::VSLIIMM, SDLoc(
Op),
Op->getVTList(),
4026 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4027 case Intrinsic::arm_mve_vsri:
4028 return DAG.
getNode(ARMISD::VSRIIMM, SDLoc(
Op),
Op->getVTList(),
4029 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4045 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4046 return DAG.
getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other,
Op.getOperand(0),
4056 }
else if (
Subtarget->preferISHSTBarriers() &&
4065 DAG.
getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4075 return Op.getOperand(0);
4078 unsigned isRead =
~Op.getConstantOperandVal(2) & 1;
4082 return Op.getOperand(0);
4084 unsigned isData =
Op.getConstantOperandVal(4);
4087 isRead = ~isRead & 1;
4088 isData = ~isData & 1;
4091 return DAG.
getNode(ARMISD::PRELOAD, dl, MVT::Other,
Op.getOperand(0),
4106 return DAG.
getStore(
Op.getOperand(0), dl, FR,
Op.getOperand(1),
4114 const SDLoc &dl)
const {
4116 ARMFunctionInfo *AFI = MF.
getInfo<ARMFunctionInfo>();
4120 RC = &ARM::tGPRRegClass;
4122 RC = &ARM::GPRRegClass;
4136 MVT::i32, dl, Root, FIN,
4142 if (!Subtarget->isLittle())
4144 return DAG.
getNode(ARMISD::VMOVDRR, dl, MVT::f64, ArgValue, ArgValue2);
4157 const Value *OrigArg,
4158 unsigned InRegsParamRecordIdx,
4159 int ArgOffset,
unsigned ArgSize)
const {
4173 ARMFunctionInfo *AFI = MF.
getInfo<ARMFunctionInfo>();
4174 unsigned RBegin, REnd;
4179 RBegin = RBeginIdx == 4 ? (unsigned)ARM::R4 :
GPRArgRegs[RBeginIdx];
4184 ArgOffset = -4 * (ARM::R4 - RBegin);
4194 for (
unsigned Reg = RBegin, i = 0;
Reg < REnd; ++
Reg, ++i) {
4198 MachinePointerInfo(OrigArg, 4 * i));
4203 if (!MemOps.
empty())
4212 unsigned TotalArgRegsSaveSize,
4213 bool ForceMutable)
const {
4215 ARMFunctionInfo *AFI = MF.
getInfo<ARMFunctionInfo>();
4224 CCInfo.
getStackSize(), std::max(4U, TotalArgRegsSaveSize));
4228bool ARMTargetLowering::splitValueIntoRegisterParts(
4230 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
4232 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4244SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4246 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
4247 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4260SDValue ARMTargetLowering::LowerFormalArguments(
4267 ARMFunctionInfo *AFI = MF.
getInfo<ARMFunctionInfo>();
4276 unsigned CurArgIdx = 0;
4288 unsigned ArgRegBegin = ARM::R4;
4289 for (
const CCValAssign &VA : ArgLocs) {
4295 if (!
Flags.isByVal())
4299 unsigned RBegin, REnd;
4301 ArgRegBegin = std::min(ArgRegBegin, RBegin);
4307 int lastInsIndex = -1;
4311 ArgRegBegin = std::min(ArgRegBegin, (
unsigned)
GPRArgRegs[RegIdx]);
4314 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4318 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4319 CCValAssign &VA = ArgLocs[i];
4320 if (Ins[VA.
getValNo()].isOrigArg()) {
4321 std::advance(CurOrigArg,
4322 Ins[VA.
getValNo()].getOrigArgIndex() - CurArgIdx);
4323 CurArgIdx = Ins[VA.
getValNo()].getOrigArgIndex();
4334 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4341 MVT::f64, dl, Chain, FIN,
4344 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4352 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4356 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4357 RC = &ARM::HPRRegClass;
4358 else if (RegVT == MVT::f32)
4359 RC = &ARM::SPRRegClass;
4360 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4361 RegVT == MVT::v4bf16)
4362 RC = &ARM::DPRRegClass;
4363 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4364 RegVT == MVT::v8bf16)
4365 RC = &ARM::QPRRegClass;
4366 else if (RegVT == MVT::i32)
4368 : &ARM::GPRRegClass;
4405 const ISD::InputArg &Arg = Ins[VA.
getValNo()];
4414 assert(VA.
getValVT() != MVT::i64 &&
"i64 should already be lowered");
4420 if (index != lastInsIndex)
4422 ISD::ArgFlagsTy
Flags = Ins[index].Flags;
4428 if (
Flags.isByVal()) {
4429 assert(Ins[index].isOrigArg() &&
4430 "Byval arguments cannot be implicit");
4434 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4468 lastInsIndex = index;
4475 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.
getStackSize(),
4476 TotalArgRegsSaveSize);
4480 "secure entry function must not be variadic", dl.
getDebugLoc()));
4501 SmallVectorImpl<ForwardedRegister> &Forwards =
4514 assert(StackAlign &&
"data layout string is missing stack alignment");
4515 StackArgSize =
alignTo(StackArgSize, *StackAlign);
4524 "secure entry function requires arguments on stack", dl.
getDebugLoc()));
4533 return CFP->getValueAPF().isPosZero();
4536 if (
Op.getOperand(1).getOpcode() == ARMISD::Wrapper) {
4537 SDValue WrapperOp =
Op.getOperand(1).getOperand(0);
4540 return CFP->getValueAPF().isPosZero();
4543 Op->getValueType(0) == MVT::f64) {
4547 if (BitcastOp->
getOpcode() == ARMISD::VMOVIMM &&
4556 if (
Op->getFlags().hasNoSignedWrap())
4572 (isIntEqualitySetCC(
CC) ||
4582 if (ST.isThumb1Only() || !
Op.hasOneUse())
4585 unsigned Opc =
Op.getOpcode();
4588 return ShiftAmt->getZExtValue() <= 31 ? 1 : 0;
4591 return ST.isThumb() ? 0 : 1;
4599 return ST.isThumb() ? 0 : 1;
4615 uint32_t MaskValue = Mask->getZExtValue();
4618 if (!
isMask_32(MaskValue) || MaskValue == 1 || MaskValue == 0xff ||
4619 MaskValue == 0xffff || MaskValue == ~0U)
4631 const SDLoc &dl)
const {
4638 for (
unsigned I = 0;
I != 2; ++
I) {
4648 unsigned C = RHSC->getZExtValue();
4706 if (Subtarget->isThumb1Only() &&
LHS->getOpcode() ==
ISD::AND &&
4710 unsigned Mask =
LHS.getConstantOperandVal(1);
4712 uint64_t RHSV = RHSC->getZExtValue();
4713 if (
isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4715 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4729 if (Subtarget->isThumb1Only() &&
LHS->getOpcode() ==
ISD::SHL &&
4732 LHS.getConstantOperandVal(1) < 31) {
4733 unsigned ShiftAmt =
LHS.getConstantOperandVal(1) + 1;
4743 unsigned CompareType;
4746 CompareType = ARMISD::CMP;
4751 CompareType = ARMISD::CMPZ;
4760 if (CompareType != ARMISD::CMPZ &&
isCMN(
RHS,
CC, DAG)) {
4761 CompareType = ARMISD::CMN;
4763 }
else if (CompareType != ARMISD::CMPZ &&
isCMN(
LHS,
CC, DAG)) {
4764 CompareType = ARMISD::CMN;
4778 if (CompareType == ARMISD::CMP)
4806 bool Signaling)
const {
4807 assert(Subtarget->hasFP64() ||
RHS.getValueType() != MVT::f64);
4813 Flags = DAG.
getNode(Signaling ? ARMISD::CMPFPEw0 : ARMISD::CMPFPw0, dl,
4822std::pair<SDValue, SDValue>
4825 assert(
Op.getValueType() == MVT::i32 &&
"Unsupported value type");
4837 switch (
Op.getOpcode()) {
4889 return std::make_pair(
Value, OverflowCmp);
4902 return Cmp.getValue(1);
4930 return DAG.
getNode(ARMISD::CMOV,
DL, VT,
Zero, One, ARMcc, Flags);
4942 EVT VT =
Op.getValueType();
4943 SDVTList VTs = DAG.
getVTList(VT, MVT::i32);
4946 switch (
Op.getOpcode()) {
4960 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Op, DAG, ARMcc);
4966 DAG.
getNode(ARMISD::CMOV, dl, MVT::i32,
4969 ARMcc, OverflowCmp);
4979 EVT VT =
Op.getValueType();
4990 switch (
Op->getOpcode()) {
4992 NewOpcode = ARMISD::UQADD8b;
4995 NewOpcode = ARMISD::QADD8b;
4998 NewOpcode = ARMISD::UQSUB8b;
5001 NewOpcode = ARMISD::QSUB8b;
5006 switch (
Op->getOpcode()) {
5008 NewOpcode = ARMISD::UQADD16b;
5011 NewOpcode = ARMISD::QADD16b;
5014 NewOpcode = ARMISD::UQSUB16b;
5017 NewOpcode = ARMISD::QSUB16b;
5025 DAG.
getNode(NewOpcode, dl, MVT::i32,
5036 unsigned Opc =
Cond.getOpcode();
5038 if (
Cond.getResNo() == 1 &&
5046 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5047 EVT VT =
Op.getValueType();
5049 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, OverflowCmp, DAG);
5057 if (
Cond.getOpcode() == ARMISD::CMOV &&
Cond.hasOneUse()) {
5058 const ConstantSDNode *CMOVTrue =
5060 const ConstantSDNode *CMOVFalse =
5063 if (CMOVTrue && CMOVFalse) {
5069 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5071 False = SelectFalse;
5072 }
else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5078 return getCMOV(dl,
Op.getValueType(),
True,
False,
Cond.getOperand(2),
5079 Cond.getOperand(3), DAG);
5089 bool &swpCmpOps,
bool &swpVselOps) {
5117 swpCmpOps = !swpCmpOps;
5118 swpVselOps = !swpVselOps;
5141 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5143 DAG.
getVTList(MVT::i32, MVT::i32), FalseVal);
5145 DAG.
getVTList(MVT::i32, MVT::i32), TrueVal);
5159 return DAG.
getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, Flags);
5180 ((
K ==
LHS &&
K == TrueVal) || (
K ==
RHS &&
K == FalseVal))) ||
5182 ((
K ==
RHS &&
K == TrueVal) || (
K ==
LHS &&
K == FalseVal)));
5203 EVT VT =
Op.getValueType();
5225 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5238 int64_t PosVal = std::max(Val1, Val2);
5239 int64_t NegVal = std::min(Val1, Val2);
5251 return DAG.
getNode(ARMISD::SSAT, dl, VT, V2Tmp,
5254 return DAG.
getNode(ARMISD::USAT, dl, VT, V2Tmp,
5286 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5291 if (*
K != KTmp ||
V != VTmp)
5302bool ARMTargetLowering::isUnsupportedFloatingType(
EVT VT)
const {
5304 return !Subtarget->hasVFP2Base();
5306 return !Subtarget->hasFP64();
5308 return !Subtarget->hasFullFP16();
5316 if (!CFVal || !CTVal || !
Subtarget->hasV8_1MMainlineOps())
5324 if (TVal == ~FVal) {
5325 Opcode = ARMISD::CSINV;
5326 }
else if (TVal == ~FVal + 1) {
5327 Opcode = ARMISD::CSNEG;
5328 }
else if (TVal + 1 == FVal) {
5329 Opcode = ARMISD::CSINC;
5330 }
else if (TVal == FVal + 1) {
5331 Opcode = ARMISD::CSINC;
5334 InvertCond = !InvertCond;
5341 if (Opcode != ARMISD::CSINC &&
5345 InvertCond = !InvertCond;
5351 if (FVal == 0 && Opcode != ARMISD::CSINC) {
5354 InvertCond = !InvertCond;
5361 EVT VT =
Op.getValueType();
5365 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->isThumb2())
5377 if (VT == MVT::i32 &&
5396 if (
Op.getValueType().isInteger()) {
5404 LHS.getValueType() ==
RHS.getValueType()) {
5405 EVT VT =
LHS.getValueType();
5411 Shift = DAG.
getNOT(dl, Shift, VT);
5423 if (
LHS.getValueType() == MVT::i32) {
5427 matchCSET(Opcode, InvertCond, TrueVal, FalseVal, Subtarget)) {
5433 EVT VT =
Op.getValueType();
5434 return DAG.
getNode(Opcode, dl, VT,
Op,
Op, ARMcc, Cmp);
5438 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5443 if (!
RHS.getNode()) {
5449 if (
LHS.getValueType() == MVT::i32) {
5460 if (Subtarget->hasFPARMv8Base() && (
TrueVal.getValueType() == MVT::f16 ||
5461 TrueVal.getValueType() == MVT::f32 ||
5462 TrueVal.getValueType() == MVT::f64)) {
5476 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Cmp, DAG);
5486 if (Subtarget->hasFPARMv8Base() &&
5488 (
TrueVal.getValueType() == MVT::f16 ||
5489 TrueVal.getValueType() == MVT::f32 ||
5490 TrueVal.getValueType() == MVT::f64)) {
5491 bool swpCmpOps =
false;
5492 bool swpVselOps =
false;
5506 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Cmp, DAG);
5509 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, Cmp, DAG);
5519 if (!
N->hasOneUse())
5522 if (!
N->getNumValues())
5524 EVT VT =
Op.getValueType();
5525 if (VT != MVT::f32 && !
Subtarget->isFPBrccSlow())
5542 return DAG.
getLoad(MVT::i32,
SDLoc(
Op), Ld->getChain(), Ld->getBasePtr(),
5543 Ld->getPointerInfo(), Ld->getAlign(),
5544 Ld->getMemOperand()->getFlags());
5560 SDValue Ptr = Ld->getBasePtr();
5562 DAG.
getLoad(MVT::i32, dl, Ld->getChain(), Ptr, Ld->getPointerInfo(),
5563 Ld->getAlign(), Ld->getMemOperand()->
getFlags());
5568 RetVal2 = DAG.
getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5569 Ld->getPointerInfo().getWithOffset(4),
5571 Ld->getMemOperand()->getFlags());
5589 bool LHSSeenZero =
false;
5591 bool RHSSeenZero =
false;
5593 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5604 if (
LHS.getValueType() == MVT::f32) {
5610 return DAG.
getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
5622 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5623 return DAG.
getNode(ARMISD::BCC_i64, dl, MVT::Other,
Ops);
5638 return DAG.
getNode(ARMISD::CMOV,
DL, MVT::i32,
Op.getOperand(0), Neg,
5657 unsigned Opc =
Cond.getOpcode();
5659 !Subtarget->isThumb1Only();
5660 if (
Cond.getResNo() == 1 &&
5670 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5675 return DAG.
getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
5690 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5695 if (!
RHS.getNode()) {
5703 unsigned Opc =
LHS.getOpcode();
5705 !Subtarget->isThumb1Only();
5717 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
LHS.getValue(0), DAG, ARMcc);
5724 return DAG.
getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
5728 if (
LHS.getValueType() == MVT::i32) {
5731 return DAG.
getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, Cmp);
5734 SDNodeFlags
Flags =
Op->getFlags();
5735 if (
Flags.hasNoNaNs() &&
5740 if (
SDValue Result = OptimizeVFPBrcond(
Op, DAG))
5754 Res = DAG.
getNode(ARMISD::BRCOND, dl, MVT::Other,
Ops);
5768 Table = DAG.
getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI);
5771 if (Subtarget->isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5776 return DAG.
getNode(ARMISD::BR2_JT, dl, MVT::Other, Chain,
5777 Addr,
Op.getOperand(2), JTI);
5781 DAG.
getLoad((EVT)MVT::i32, dl, Chain, Addr,
5785 return DAG.
getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
5788 DAG.
getLoad(PTy, dl, Chain, Addr,
5791 return DAG.
getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
5796 EVT VT =
Op.getValueType();
5799 if (
Op.getValueType().getVectorElementType() == MVT::i32) {
5800 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
5808 const EVT OpTy =
Op.getOperand(0).getValueType();
5809 if (
OpTy == MVT::v4f32)
5811 else if (
OpTy == MVT::v4f16 && HasFullFP16)
5813 else if (
OpTy == MVT::v8f16 && HasFullFP16)
5818 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5821 Op = DAG.
getNode(
Op.getOpcode(), dl, NewTy,
Op.getOperand(0));
5826 EVT VT =
Op.getValueType();
5830 bool IsStrict =
Op->isStrictFPOpcode();
5831 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
5833 if (isUnsupportedFloatingType(SrcVal.
getValueType())) {
5846 std::tie(Result, Chain) =
makeLibCall(DAG, LC,
Op.getValueType(), SrcVal,
5847 CallOptions, Loc, Chain);
5856 EVT VT =
Op.getValueType();
5858 EVT FromVT =
Op.getOperand(0).getValueType();
5860 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5862 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5865 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5868 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5871 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5875 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5892 EVT VT =
Op.getValueType();
5895 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
5901 assert((
Op.getOperand(0).getValueType() == MVT::v4i16 ||
5902 Op.getOperand(0).getValueType() == MVT::v8i16) &&
5903 "Invalid type for custom lowering!");
5908 if (VT == MVT::v4f32)
5909 DestVecType = MVT::v4i32;
5910 else if (VT == MVT::v4f16 && HasFullFP16)
5911 DestVecType = MVT::v4i16;
5912 else if (VT == MVT::v8f16 && HasFullFP16)
5913 DestVecType = MVT::v8i16;
5919 switch (
Op.getOpcode()) {
5931 Op = DAG.
getNode(CastOpc, dl, DestVecType,
Op.getOperand(0));
5936 EVT VT =
Op.getValueType();
5940 bool IsStrict =
Op->isStrictFPOpcode();
5941 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
5943 if (isUnsupportedFloatingType(VT)) {
5954 std::tie(Result, Chain) =
makeLibCall(DAG, LC,
Op.getValueType(), SrcVal,
5955 CallOptions, Loc, Chain);
5967 EVT VT =
Op.getValueType();
5971 bool UseNEON = !InGPR && Subtarget->hasNEON();
5978 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
5985 if (SrcVT == MVT::f32) {
5988 Tmp1 = DAG.
getNode(ARMISD::VSHLIMM, dl, OpVT,
5991 }
else if (VT == MVT::f32)
5992 Tmp1 = DAG.
getNode(ARMISD::VSHRuIMM, dl, MVT::v1i64,
6007 if (VT == MVT::f32) {
6019 if (SrcVT == MVT::f64)
6028 if (VT == MVT::f32) {
6041 return DAG.
getNode(ARMISD::VMOVDRR, dl, MVT::f64,
Lo,
Hi);
6049 EVT VT =
Op.getValueType();
6051 unsigned Depth =
Op.getConstantOperandVal(0);
6053 SDValue FrameAddr = LowerFRAMEADDR(
Op, DAG);
6057 MachinePointerInfo());
6066 const ARMBaseRegisterInfo &ARI =
6067 *
static_cast<const ARMBaseRegisterInfo*
>(RegInfo);
6072 EVT VT =
Op.getValueType();
6074 unsigned Depth =
Op.getConstantOperandVal(0);
6079 MachinePointerInfo());
6087 return StringSwitch<Register>(
RegName)
6088 .Case(
"sp", ARM::SP)
6099 assert(
N->getValueType(0) == MVT::i64
6100 &&
"ExpandREAD_REGISTER called for non-i64 type result.");
6103 DAG.
getVTList(MVT::i32, MVT::i32, MVT::Other),
6143 const APInt &APIntIndex = Index->getAPIntValue();
6145 NewIndex *= APIntIndex;
6174 EVT SrcVT =
Op.getValueType();
6175 EVT DstVT =
N->getValueType(0);
6177 if ((SrcVT == MVT::f16 || SrcVT == MVT::bf16) &&
6178 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6181 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6182 (DstVT == MVT::f16 || DstVT == MVT::bf16)) {
6184 if (Subtarget->hasFullFP16())
6185 return DAG.
getNode(ARMISD::VMOVhr, dl, DstVT, Ext);
6186 return DAG.
getNode(ARMISD::VMOVsr, dl, DstVT, Ext);
6189 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6190 (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
6191 if (Subtarget->hasFullFP16() && !Subtarget->hasBF16())
6198 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6210 DAG.
getNode(ARMISD::VMOVDRR, dl, MVT::f64,
Lo,
Hi));
6218 Cvt = DAG.
getNode(ARMISD::VMOVRRD, dl,
6220 DAG.
getNode(ARMISD::VREV64, dl, SrcVT,
Op));
6222 Cvt = DAG.
getNode(ARMISD::VMOVRRD, dl,
6242 SDValue Vmov = DAG.
getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal);
6251 EVT VT =
Op.getValueType();
6273 DAG.
getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift, ARMcc, CmpLo);
6283 DAG.
getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, ARMcc, CmpHi);
6294 EVT VT =
Op.getValueType();
6315 DAG.
getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, ARMcc, CmpHi);
6336 DAG.
getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6388 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6416 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6446 EVT VT =
N->getValueType(0);
6447 if (VT.
isVector() && ST->hasNEON()) {
6467 unsigned NumBits =
ElemTy.getSizeInBits();
6469 DAG.
getNode(ARMISD::VMOVIMM, dl, VT,
6479 if (
ElemTy == MVT::i64) {
6492 if (!ST->hasV6T2Ops())
6501 EVT VT =
N->getValueType(0);
6504 assert(ST->hasNEON() &&
"Custom ctpop lowering requires NEON.");
6505 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6506 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6507 "Unexpected type for custom ctpop lowering");
6515 unsigned EltSize = 8;
6538 Op =
Op.getOperand(0);
6540 APInt SplatBits, SplatUndef;
6541 unsigned SplatBitSize;
6544 !BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6546 SplatBitSize > ElementBits)
6557 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6561 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6572 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6577 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6578 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6587 EVT VT =
N->getValueType(0);
6602 return DAG.
getNode(ARMISD::VSHLIMM, dl, VT,
N->getOperand(0),
6604 return DAG.
getNode(ARMISD::VSHLu, dl, VT,
N->getOperand(0),
6609 "unexpected vector shift opcode");
6611 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
6612 unsigned VShiftOpc =
6613 (
N->getOpcode() ==
ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
6614 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
6620 EVT ShiftVT =
N->getOperand(1).getValueType();
6623 unsigned VShiftOpc =
6624 (
N->getOpcode() ==
ISD::SRA ? ARMISD::VSHLs : ARMISD::VSHLu);
6625 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0), NegatedCount);
6630 EVT VT =
N->getValueType(0);
6639 "Unknown shift to lower!");
6641 unsigned ShOpc =
N->getOpcode();
6642 if (ST->hasMVEIntegerOps()) {
6644 unsigned ShPartsOpc = ARMISD::LSLL;
6665 ShPartsOpc = ARMISD::LSRL;
6667 ShPartsOpc = ARMISD::ASRL;
6672 DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6686 if (ST->isThumb1Only())
6691 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6695 unsigned Opc =
N->getOpcode() ==
ISD::SRL ? ARMISD::LSRS1 : ARMISD::ASRS1;
6699 Lo = DAG.
getNode(ARMISD::RRX, dl, MVT::i32,
Lo,
Hi.getValue(1));
6707 bool Invert =
false;
6714 EVT VT =
Op.getValueType();
6722 assert(ST->hasMVEIntegerOps() &&
6723 "No hardware support for integer vector comparison!");
6725 if (
Op.getValueType().getVectorElementType() != MVT::i1)
6746 SDValue Reversed = DAG.
getNode(ARMISD::VREV64, dl, SplitVT, Cmp);
6750 Merged = DAG.
getNOT(dl, Merged, CmpVT);
6760 switch (SetCCOpcode) {
6764 if (ST->hasMVEFloatOps()) {
6767 Invert =
true; [[fallthrough]];
6792 Result = DAG.
getNOT(dl, Result, VT);
6795 case ISD::SETUO: Invert =
true; [[fallthrough]];
6804 Result = DAG.
getNOT(dl, Result, VT);
6810 switch (SetCCOpcode) {
6813 if (ST->hasMVEIntegerOps()) {
6816 Invert =
true; [[fallthrough]];
6838 if (AndOp.getNode() && AndOp.getOpcode() ==
ISD::BITCAST)
6841 if (AndOp.getNode() && AndOp.getOpcode() ==
ISD::AND) {
6846 Result = DAG.
getNOT(dl, Result, VT);
6871 Result = DAG.
getNode(ARMISD::VCMPZ, dl, CmpVT, Op0,
6874 Result = DAG.
getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6880 Result = DAG.
getNOT(dl, Result, VT);
6889 assert(
LHS.getSimpleValueType().isInteger() &&
"SETCCCARRY is integer only.");
6906 return DAG.
getNode(ARMISD::CMOV,
DL,
Op.getValueType(), FVal, TVal, ARMcc,
6917 unsigned OpCmode,
Imm;
6928 switch (SplatBitSize) {
6933 assert((SplatBits & ~0xff) == 0 &&
"one byte splat value is too big");
6936 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
6941 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
6942 if ((SplatBits & ~0xff) == 0) {
6948 if ((SplatBits & ~0xff00) == 0) {
6951 Imm = SplatBits >> 8;
6961 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
6962 if ((SplatBits & ~0xff) == 0) {
6968 if ((SplatBits & ~0xff00) == 0) {
6971 Imm = SplatBits >> 8;
6974 if ((SplatBits & ~0xff0000) == 0) {
6977 Imm = SplatBits >> 16;
6980 if ((SplatBits & ~0xff000000) == 0) {
6983 Imm = SplatBits >> 24;
6990 if ((SplatBits & ~0xffff) == 0 &&
6991 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
6994 Imm = SplatBits >> 8;
7002 if ((SplatBits & ~0xffffff) == 0 &&
7003 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
7006 Imm = SplatBits >> 16;
7022 unsigned ImmMask = 1;
7024 for (
int ByteNum = 0; ByteNum < 8; ++ByteNum) {
7025 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
7027 }
else if ((SplatBits & BitMask) != 0) {
7036 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
7050 EVT VT =
Op.getValueType();
7051 bool IsDouble = (VT == MVT::f64);
7057 if (
ST->genExecuteOnly()) {
7059 assert((!
ST->isThumb1Only() ||
ST->hasV8MBaselineOps()) &&
7060 "Unexpected architecture");
7078 return DAG.
getNode(ARMISD::VMOVsr,
DL, VT,
7083 if (!
ST->hasVFP3Base())
7088 if (IsDouble && !Subtarget->hasFP64())
7095 if (IsDouble || !
ST->useNEONForSinglePrecisionFP()) {
7113 if (!
ST->hasNEON() || (!IsDouble && !
ST->useNEONForSinglePrecisionFP()))
7122 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7176 unsigned ExpectedElt =
Imm;
7177 for (
unsigned i = 1; i < NumElts; ++i) {
7181 if (ExpectedElt == NumElts)
7184 if (M[i] < 0)
continue;
7185 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7193 bool &ReverseVEXT,
unsigned &
Imm) {
7195 ReverseVEXT =
false;
7206 unsigned ExpectedElt =
Imm;
7207 for (
unsigned i = 1; i < NumElts; ++i) {
7211 if (ExpectedElt == NumElts * 2) {
7216 if (M[i] < 0)
continue;
7217 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7232 return VT == MVT::v8i8 && M.size() == 8;
7238 unsigned &WhichResult,
7241 if (
isVTRNMask(ShuffleMask, VT, WhichResult))
7242 return ARMISD::VTRN;
7243 if (
isVUZPMask(ShuffleMask, VT, WhichResult))
7244 return ARMISD::VUZP;
7245 if (
isVZIPMask(ShuffleMask, VT, WhichResult))
7246 return ARMISD::VZIP;
7250 return ARMISD::VTRN;
7252 return ARMISD::VUZP;
7254 return ARMISD::VZIP;
7263 if (NumElts != M.size())
7267 for (
unsigned i = 0; i != NumElts; ++i)
7268 if (M[i] >= 0 && M[i] != (
int) (NumElts - 1 - i))
7277 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7285 int Ofs = Top ? 1 : 0;
7286 int Upper = SingleSource ? 0 : NumElts;
7287 for (
int i = 0, e = NumElts / 2; i != e; ++i) {
7288 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7290 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs +
Upper)
7299 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7308 unsigned Offset = Top ? 0 : 1;
7309 unsigned N = SingleSource ? 0 : NumElts;
7310 for (
unsigned i = 0; i < NumElts; i += 2) {
7311 if (M[i] >= 0 && M[i] != (
int)i)
7313 if (M[i + 1] >= 0 && M[i + 1] != (
int)(
N + i +
Offset))
7322 if (NumElts != M.size())
7330 unsigned Off0 = rev ? NumElts / 2 : 0;
7331 unsigned Off1 = rev ? 0 : NumElts / 2;
7332 for (
unsigned i = 0; i < NumElts; i += 2) {
7333 if (M[i] >= 0 && M[i] != (
int)(Off0 + i / 2))
7335 if (M[i + 1] >= 0 && M[i + 1] != (
int)(Off1 + i / 2))
7351 if (!ST->hasMVEFloatOps())
7356 if (VT != MVT::v8f16)
7377 for (
unsigned i = 1; i < 4; i++) {
7392 return DAG.
getNode(ARMISD::VCVTN, dl, VT, N1, Op1,
7404 if (!ST->hasMVEFloatOps())
7409 if (VT != MVT::v4f32)
7425 for (
unsigned i = 1; i < 4; i++) {
7436 return DAG.
getNode(ARMISD::VCVTL, dl, VT, Op0,
7448 Val =
N->getAsZExtVal();
7450 if (ST->isThumb1Only()) {
7451 if (Val <= 255 || ~Val <= 255)
7463 EVT VT =
Op.getValueType();
7465 assert(ST->hasMVEIntegerOps() &&
"LowerBUILD_VECTOR_i1 called without MVE!");
7469 unsigned BitsPerBool;
7473 }
else if (NumElts == 4) {
7476 }
else if (NumElts == 8) {
7479 }
else if (NumElts == 16) {
7490 return U.get().isUndef() || U.get() == FirstOp;
7494 return DAG.
getNode(ARMISD::PREDICATE_CAST, dl,
Op.getValueType(), Ext);
7498 unsigned Bits32 = 0;
7499 for (
unsigned i = 0; i < NumElts; ++i) {
7503 bool BitSet =
V.isUndef() ?
false :
V->getAsZExtVal();
7505 Bits32 |= BoolMask << (i * BitsPerBool);
7511 for (
unsigned i = 0; i < NumElts; ++i) {
7524 if (!ST->hasMVEIntegerOps())
7528 EVT VT =
Op.getValueType();
7538 if (
N != 1 &&
N != 2 &&
N != 4 &&
N != 8)
7542 for (
unsigned I = 2;
I < NumElts;
I++) {
7558 switch (
N->getOpcode()) {
7569 return N->getOperand(1).getNode() ==
Op;
7571 switch (
N->getConstantOperandVal(0)) {
7572 case Intrinsic::arm_mve_add_predicated:
7573 case Intrinsic::arm_mve_mul_predicated:
7574 case Intrinsic::arm_mve_qadd_predicated:
7575 case Intrinsic::arm_mve_vhadd:
7576 case Intrinsic::arm_mve_hadd_predicated:
7577 case Intrinsic::arm_mve_vqdmulh:
7578 case Intrinsic::arm_mve_qdmulh_predicated:
7579 case Intrinsic::arm_mve_vqrdmulh:
7580 case Intrinsic::arm_mve_qrdmulh_predicated:
7581 case Intrinsic::arm_mve_vqdmull:
7582 case Intrinsic::arm_mve_vqdmull_predicated:
7584 case Intrinsic::arm_mve_sub_predicated:
7585 case Intrinsic::arm_mve_qsub_predicated:
7586 case Intrinsic::arm_mve_vhsub:
7587 case Intrinsic::arm_mve_hsub_predicated:
7588 return N->getOperand(2).getNode() ==
Op;
7603 EVT VT =
Op.getValueType();
7611 APInt SplatBits, SplatUndef;
7612 unsigned SplatBitSize;
7614 if (BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7621 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7623 [BVN](
const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
7624 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7625 : SplatBitSize == 16 ? MVT::v8i16
7629 return DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup);
7632 if ((
ST->hasNEON() && SplatBitSize <= 64) ||
7633 (
ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7638 SplatBitSize, DAG, dl, VmovVT, VT,
VMOVModImm);
7642 return DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vmov);
7646 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7648 NegatedImm, SplatUndef.
getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
7652 return DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vmov);
7656 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7660 return DAG.
getNode(ARMISD::VMOVFPIMM, dl, VT, Val);
7666 if (
ST->hasMVEIntegerOps() &&
7667 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
7668 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7669 : SplatBitSize == 16 ? MVT::v8i16
7673 return DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup);
7686 bool isOnlyLowElement =
true;
7687 bool usesOnlyOneValue =
true;
7688 bool hasDominantValue =
false;
7693 DenseMap<SDValue, unsigned> ValueCounts;
7695 for (
unsigned i = 0; i < NumElts; ++i) {
7700 isOnlyLowElement =
false;
7704 unsigned &
Count = ValueCounts[
V];
7707 if (++
Count > (NumElts / 2)) {
7708 hasDominantValue =
true;
7712 if (ValueCounts.
size() != 1)
7713 usesOnlyOneValue =
false;
7714 if (!
Value.getNode() && !ValueCounts.
empty())
7717 if (ValueCounts.
empty())
7723 (VT != MVT::v8f16 ||
ST->hasFullFP16()))
7730 if (hasDominantValue && EltSize <= 32) {
7739 ConstantSDNode *constIndex;
7746 if (VT !=
Value->getOperand(0).getValueType()) {
7749 N = DAG.
getNode(ARMISD::VDUPLANE, dl, VT,
7754 N = DAG.
getNode(ARMISD::VDUPLANE, dl, VT,
7759 if (!usesOnlyOneValue) {
7762 for (
unsigned I = 0;
I < NumElts; ++
I) {
7767 Ops.push_back(
Op.getOperand(
I));
7777 assert(FVT == MVT::f32 || FVT == MVT::f16);
7778 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
7779 for (
unsigned i = 0; i < NumElts; ++i)
7784 Val = LowerBUILD_VECTOR(Val, DAG, ST);
7788 if (usesOnlyOneValue) {
7791 return DAG.
getNode(ARMISD::VDUP, dl, VT, Val);
7815 if (
ST->hasNEON() && VT.
is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
7835 if (EltSize >= 32) {
7841 for (
unsigned i = 0; i < NumElts; ++i)
7854 (VT == MVT::v8f16 && !
ST->hasFullFP16())) {
7856 for (
unsigned i = 0 ; i < NumElts; ++i) {
7875 EVT VT =
Op.getValueType();
7878 struct ShuffleSourceInfo {
7880 unsigned MinElt = std::numeric_limits<unsigned>::max();
7881 unsigned MaxElt = 0;
7891 int WindowScale = 1;
7893 ShuffleSourceInfo(
SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
7901 for (
unsigned i = 0; i < NumElts; ++i) {
7916 SDValue SourceVec =
V.getOperand(0);
7918 if (Source == Sources.
end())
7922 unsigned EltNo =
V.getConstantOperandVal(1);
7929 if (Sources.
size() > 2)
7935 for (
auto &Source : Sources) {
7936 EVT SrcEltTy =
Source.Vec.getValueType().getVectorElementType();
7937 if (SrcEltTy.
bitsLT(SmallestEltTy))
7938 SmallestEltTy = SrcEltTy;
7940 unsigned ResMultiplier =
7948 for (
auto &Src : Sources) {
7949 EVT SrcVT = Src.ShuffleVec.getValueType();
7953 if (SrcVTSize == VTSize)
7962 if (SrcVTSize < VTSize) {
7963 if (2 * SrcVTSize != VTSize)
7969 DAG.
getUNDEF(Src.ShuffleVec.getValueType()));
7973 if (SrcVTSize != 2 * VTSize)
7976 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
7981 if (Src.MinElt >= NumSrcElts) {
7986 Src.WindowBase = -NumSrcElts;
7987 }
else if (Src.MaxElt < NumSrcElts) {
8001 Src.ShuffleVec = DAG.
getNode(ARMISD::VEXT, dl, DestVT, VEXTSrc1,
8004 Src.WindowBase = -Src.MinElt;
8011 for (
auto &Src : Sources) {
8012 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8013 if (SrcEltTy == SmallestEltTy)
8016 Src.ShuffleVec = DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, ShuffleVT, Src.ShuffleVec);
8018 Src.WindowBase *= Src.WindowScale;
8023 for (
auto Src : Sources)
8024 assert(Src.ShuffleVec.getValueType() == ShuffleVT);
8032 if (
Entry.isUndef())
8041 EVT OrigEltTy =
Entry.getOperand(0).getValueType().getVectorElementType();
8044 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8048 int *LaneMask = &
Mask[i * ResMultiplier];
8050 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8051 ExtractBase += NumElts * (Src - Sources.begin());
8052 for (
int j = 0;
j < LanesDefined; ++
j)
8053 LaneMask[j] = ExtractBase + j;
8059 assert(Sources.size() <= 2 &&
"Too many sources!");
8062 for (
unsigned i = 0; i < Sources.size(); ++i)
8069 return DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Shuffle);
8091 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8111 unsigned PFIndexes[4];
8112 for (
unsigned i = 0; i != 4; ++i) {
8116 PFIndexes[i] = M[i];
8120 unsigned PFTableIndex =
8121 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8123 unsigned Cost = (PFEntry >> 30);
8129 bool ReverseVEXT, isV_UNDEF;
8130 unsigned Imm, WhichResult;
8133 if (EltSize >= 32 ||
8140 else if (Subtarget->hasNEON() &&
8145 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8148 else if (Subtarget->hasMVEIntegerOps() &&
8152 else if (Subtarget->hasMVEIntegerOps() &&
8166 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8167 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8168 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8171 if (LHSID == (1*9+2)*9+3)
return LHS;
8172 assert(LHSID == ((4*9+5)*9+6)*9+7 &&
"Illegal OP_COPY!");
8186 return DAG.
getNode(ARMISD::VREV64, dl, VT, OpLHS);
8189 return DAG.
getNode(ARMISD::VREV32, dl, VT, OpLHS);
8192 return DAG.
getNode(ARMISD::VREV16, dl, VT, OpLHS);
8197 return DAG.
getNode(ARMISD::VDUPLANE, dl, VT,
8202 return DAG.
getNode(ARMISD::VEXT, dl, VT,
8229 for (
int I : ShuffleMask)
8233 return DAG.
getNode(ARMISD::VTBL1,
DL, MVT::v8i8,
V1,
8236 return DAG.
getNode(ARMISD::VTBL2,
DL, MVT::v8i8,
V1, V2,
8242 EVT VT =
Op.getValueType();
8244 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8245 "Expect an v8i16/v16i8 type");
8251 std::vector<int> NewMask;
8255 NewMask.push_back(i);
8285 AllZeroes = DAG.
getNode(ARMISD::VMOVIMM, dl, MVT::v16i8, AllZeroes);
8295 if (VT != MVT::v16i1)
8296 RecastV1 = DAG.
getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1,
Pred);
8311 EVT VT =
Op.getValueType();
8315 assert(ST->hasMVEIntegerOps() &&
8316 "No support for vector shuffle of boolean predicates");
8326 return DAG.
getNode(ARMISD::PREDICATE_CAST, dl, VT, srl);
8342 "Expected identical vector type in expanded i1 shuffle!");
8346 PredAsVector2, ShuffleMask);
8351 if (VT == MVT::v2i1) {
8352 SDValue BC = DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Shuffled);
8355 return DAG.
getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
8357 return DAG.
getNode(ARMISD::VCMPZ, dl, VT, Shuffled,
8368 EVT VT =
Op.getValueType();
8372 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8373 "Unexpected vector type");
8375 int QuarterSize = NumElts / 4;
8384 for (
int i = 0; i <
Length; i++) {
8385 if (ShuffleMask[Start + i] >= 0) {
8386 if (ShuffleMask[Start + i] %
Length != i)
8388 MovIdx = ShuffleMask[Start + i] /
Length;
8396 for (
int i = 1; i <
Length; i++) {
8397 if (ShuffleMask[Start + i] >= 0 &&
8398 (ShuffleMask[Start + i] /
Length != MovIdx ||
8399 ShuffleMask[Start + i] %
Length != i))
8405 for (
int Part = 0; Part < 4; ++Part) {
8407 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8421 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8426 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8428 for (
int Part = 0; Part < 4; ++Part)
8429 for (
int i = 0; i < QuarterSize; i++)
8431 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8433 VT, dl,
Op->getOperand(0),
Op->getOperand(1), NewShuffleMask);
8436 for (
int Part = 0; Part < 4; ++Part)
8452 EVT VT =
Op.getValueType();
8464 for (
int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8468 if (Mask[i] != i + BaseOffset) {
8469 if (OffElement == -1)
8475 return NonUndef > 2 && OffElement != -1;
8479 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8481 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8492 ShuffleMask[OffElement] < (
int)NumElts ?
V1 : V2,
8503 EVT VT =
Op.getValueType();
8507 if (ST->hasMVEIntegerOps() && EltSize == 1)
8518 if (EltSize <= 32) {
8522 if (Lane == -1) Lane = 0;
8526 return DAG.
getNode(ARMISD::VDUP, dl, VT,
V1.getOperand(0));
8533 bool IsScalarToVector =
true;
8534 for (
unsigned i = 1, e =
V1.getNumOperands(); i != e; ++i)
8535 if (!
V1.getOperand(i).isUndef()) {
8536 IsScalarToVector =
false;
8539 if (IsScalarToVector)
8540 return DAG.
getNode(ARMISD::VDUP, dl, VT,
V1.getOperand(0));
8542 return DAG.
getNode(ARMISD::VDUPLANE, dl, VT,
V1,
8546 bool ReverseVEXT =
false;
8548 if (ST->hasNEON() &&
isVEXTMask(ShuffleMask, VT, ReverseVEXT,
Imm)) {
8551 return DAG.
getNode(ARMISD::VEXT, dl, VT,
V1, V2,
8556 return DAG.
getNode(ARMISD::VREV64, dl, VT,
V1);
8558 return DAG.
getNode(ARMISD::VREV32, dl, VT,
V1);
8560 return DAG.
getNode(ARMISD::VREV16, dl, VT,
V1);
8572 unsigned WhichResult = 0;
8573 bool isV_UNDEF =
false;
8574 if (ST->hasNEON()) {
8576 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8583 if (ST->hasMVEIntegerOps()) {
8585 return DAG.
getNode(ARMISD::VMOVN, dl, VT, V2,
V1,
8588 return DAG.
getNode(ARMISD::VMOVN, dl, VT,
V1, V2,
8618 }) &&
"Unexpected shuffle index into UNDEF operand!");
8621 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
8624 assert((WhichResult == 0) &&
8625 "In-place shuffle of concat can only have one result!");
8634 if (ST->hasMVEIntegerOps() && EltSize <= 32 &&
8635 (ST->hasFullFP16() || VT != MVT::v8f16)) {
8639 for (
bool Top : {
false,
true}) {
8640 for (
bool SingleSource : {
false,
true}) {
8641 if (
isTruncMask(ShuffleMask, VT, Top, SingleSource)) {
8646 SingleSource ?
V1 : V2);
8662 unsigned PFIndexes[4];
8663 for (
unsigned i = 0; i != 4; ++i) {
8664 if (ShuffleMask[i] < 0)
8667 PFIndexes[i] = ShuffleMask[i];
8671 unsigned PFTableIndex =
8672 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8674 unsigned Cost = (PFEntry >> 30);
8680 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8681 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8691 if (EltSize >= 32) {
8699 for (
unsigned i = 0; i < NumElts; ++i) {
8700 if (ShuffleMask[i] < 0)
8704 ShuffleMask[i] < (
int)NumElts ?
V1 : V2,
8712 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8716 if (ST->hasNEON() && VT == MVT::v8i8)
8720 if (ST->hasMVEIntegerOps())
8725 if (VT == MVT::v8f16 && !ST->hasFullFP16()) {
8727 DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v8i16,
Op.getOperand(0));
8729 DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v8i16,
Op.getOperand(1));
8731 return DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Shuf);
8739 EVT VecVT =
Op.getOperand(0).getValueType();
8742 assert(ST->hasMVEIntegerOps() &&
8743 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
8746 DAG.
getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32,
Op->getOperand(0));
8747 unsigned Lane =
Op.getConstantOperandVal(2);
8748 unsigned LaneWidth =
8750 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
8755 return DAG.
getNode(ARMISD::PREDICATE_CAST, dl,
Op.getValueType(), BFI);
8768 if (Subtarget->hasMVEIntegerOps() &&
8769 Op.getValueType().getScalarSizeInBits() == 1)
8793 IVecIn, IElt, Lane);
8802 EVT VecVT =
Op.getOperand(0).getValueType();
8805 assert(ST->hasMVEIntegerOps() &&
8806 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
8809 DAG.
getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32,
Op->getOperand(0));
8810 unsigned Lane =
Op.getConstantOperandVal(1);
8811 unsigned LaneWidth =
8833 return DAG.
getNode(ARMISD::VGETLANEu, dl, MVT::i32, Vec, Lane);
8842 assert(
Op.getValueType().getScalarSizeInBits() == 1 &&
8843 "Unexpected custom CONCAT_VECTORS lowering");
8845 "Unexpected custom CONCAT_VECTORS lowering");
8846 assert(ST->hasMVEIntegerOps() &&
8847 "CONCAT_VECTORS lowering only supported for MVE");
8850 EVT Op1VT =
V1.getValueType();
8851 EVT Op2VT = V2.getValueType();
8852 assert(Op1VT == Op2VT &&
"Operand types don't match!");
8853 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
8854 "Unexpected i1 concat operations!");
8867 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
8872 return DAG.
getNode(ARMISD::VCMPZ, dl, VT, ConVec,
8881 auto ExtractInto = [&DAG, &dl](
SDValue NewV,
SDValue ConVec,
unsigned &j) {
8882 EVT NewVT = NewV.getValueType();
8883 EVT ConcatVT = ConVec.getValueType();
8884 unsigned ExtScale = 1;
8885 if (NewVT == MVT::v2f64) {
8886 NewV = DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, NewV);
8899 ConVec = ExtractInto(NewV1, ConVec, j);
8900 ConVec = ExtractInto(NewV2, ConVec, j);
8904 return DAG.
getNode(ARMISD::VCMPZ, dl, VT, ConVec,
8910 while (ConcatOps.
size() > 1) {
8911 for (
unsigned I = 0,
E = ConcatOps.
size();
I !=
E;
I += 2) {
8914 ConcatOps[
I / 2] = ConcatPair(
V1, V2);
8918 return ConcatOps[0];
8923 EVT VT =
Op->getValueType(0);
8929 assert(
Op.getValueType().is128BitVector() &&
Op.getNumOperands() == 2 &&
8930 "unexpected CONCAT_VECTORS");
8951 EVT VT =
Op.getValueType();
8952 EVT Op1VT =
V1.getValueType();
8957 "Unexpected custom EXTRACT_SUBVECTOR lowering");
8958 assert(ST->hasMVEIntegerOps() &&
8959 "EXTRACT_SUBVECTOR lowering only supported for MVE");
8969 EVT SubVT = MVT::v4i32;
8971 for (
unsigned i = Index, j = 0; i < (Index + NumElts); i++, j += 2) {
8981 return DAG.
getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
8986 for (
unsigned i = Index, j = 0; i < (Index + NumElts); i++, j++) {
8995 return DAG.
getNode(ARMISD::VCMPZ, dl, VT, SubVec,
9002 assert(ST->hasMVEIntegerOps() &&
"Expected MVE!");
9003 EVT VT =
N->getValueType(0);
9004 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9005 "Expected a vector i1 type!");
9007 EVT FromVT =
Op.getValueType();
9021 EVT ToVT =
N->getValueType(0);
9064 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9066 EVT FromVT =
N->getOperand(0).getValueType();
9067 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9083 EVT ToVT =
N->getValueType(0);
9084 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9087 EVT FromVT =
Op.getValueType();
9088 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9102 Ext = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext);
9103 Ext1 = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext1);
9115 EVT VT =
N->getValueType(0);
9117 SDNode *BVN =
N->getOperand(0).getNode();
9122 unsigned HiElt = 1 - LoElt;
9127 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9143 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
9144 SDNode *Elt =
N->getOperand(i).getNode();
9147 unsigned HalfSize = EltSize / 2;
9149 if (!
isIntN(HalfSize,
C->getSExtValue()))
9152 if (!
isUIntN(HalfSize,
C->getZExtValue()))
9191 switch (OrigSimpleTy) {
9207 unsigned ExtOpcode) {
9230 if (ExtendedTy == LD->getMemoryVT())
9231 return DAG.
getLoad(LD->getMemoryVT(),
SDLoc(LD), LD->getChain(),
9232 LD->getBasePtr(), LD->getPointerInfo(), LD->getAlign(),
9233 LD->getMemOperand()->getFlags());
9239 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
9240 LD->getMemoryVT(), LD->getAlign(),
9241 LD->getMemOperand()->getFlags());
9254 N->getOperand(0)->getValueType(0),
9260 "Expected extending load");
9266 DAG.
getNode(Opcode,
SDLoc(newLoad), LD->getValueType(0), newLoad);
9275 SDNode *BVN =
N->getOperand(0).getNode();
9277 BVN->
getValueType(0) == MVT::v4i32 &&
"expected v4i32 BUILD_VECTOR");
9285 EVT VT =
N->getValueType(0);
9291 for (
unsigned i = 0; i != NumElts; ++i) {
9292 const APInt &CInt =
N->getConstantOperandAPInt(i);
9301 unsigned Opcode =
N->getOpcode();
9303 SDNode *N0 =
N->getOperand(0).getNode();
9304 SDNode *N1 =
N->getOperand(1).getNode();
9312 unsigned Opcode =
N->getOpcode();
9314 SDNode *N0 =
N->getOperand(0).getNode();
9315 SDNode *N1 =
N->getOperand(1).getNode();
9325 EVT VT =
Op.getValueType();
9327 "unexpected type for custom-lowering ISD::MUL");
9328 SDNode *N0 =
Op.getOperand(0).getNode();
9329 SDNode *N1 =
Op.getOperand(1).getNode();
9330 unsigned NewOpc = 0;
9334 if (isN0SExt && isN1SExt)
9335 NewOpc = ARMISD::VMULLs;
9339 if (isN0ZExt && isN1ZExt)
9340 NewOpc = ARMISD::VMULLu;
9341 else if (isN1SExt || isN1ZExt) {
9345 NewOpc = ARMISD::VMULLs;
9348 NewOpc = ARMISD::VMULLu;
9352 NewOpc = ARMISD::VMULLu;
9358 if (VT == MVT::v2i64)
9375 "unexpected types for extended operands to VMULL");
9376 return DAG.
getNode(NewOpc,
DL, VT, Op0, Op1);
9411 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9445 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9448 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9469 EVT VT =
Op.getValueType();
9470 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9471 "unexpected type for custom-lowering ISD::SDIV");
9478 if (VT == MVT::v8i8) {
9506 EVT VT =
Op.getValueType();
9507 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9508 "unexpected type for custom-lowering ISD::UDIV");
9515 if (VT == MVT::v8i8) {
9554 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9557 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9561 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9581 unsigned Opcode,
bool IsSigned) {
9582 EVT VT0 =
Op.getValue(0).getValueType();
9583 EVT VT1 =
Op.getValue(1).getValueType();
9585 bool InvertCarry = Opcode == ARMISD::SUBE;
9605 EVT VT =
Op.getValueType();
9606 assert((VT == MVT::i32 || VT == MVT::i64) &&
9607 "unexpected type for custom lowering DIV");
9613 LC = VT == MVT::i32 ? RTLIB::SDIVREM_I32 : RTLIB::SDIVREM_I64;
9615 LC = VT == MVT::i32 ? RTLIB::UDIVREM_I32 : RTLIB::UDIVREM_I64;
9622 for (
auto AI : {1, 0}) {
9624 Args.emplace_back(Operand,
9641ARMTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
9649 const bool MinSize =
ST.hasMinSize();
9650 const bool HasDivide =
ST.isThumb() ?
ST.hasDivideInThumbMode()
9651 :
ST.hasDivideInARMMode();
9655 if (
N->getOperand(0).getValueType().isVector())
9660 if (!(MinSize && HasDivide))
9673 if (Divisor.
sgt(128))
9681 assert(
Op.getValueType() == MVT::i32 &&
9682 "unexpected type for custom lowering DIV");
9685 SDValue DBZCHK = DAG.
getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other,
9688 return LowerWindowsDIVLibCall(
Op, DAG,
Signed, DBZCHK);
9694 if (
N->getValueType(0) == MVT::i32)
9695 return DAG.
getNode(ARMISD::WIN__DBZCHK,
DL, MVT::Other, InChain,
Op);
9698 return DAG.
getNode(ARMISD::WIN__DBZCHK,
DL, MVT::Other, InChain,
9702void ARMTargetLowering::ExpandDIV_Windows(
9707 assert(
Op.getValueType() == MVT::i64 &&
9708 "unexpected type for custom lowering DIV");
9723std::pair<SDValue, SDValue>
9724ARMTargetLowering::LowerAEABIUnalignedLoad(
SDValue Op,
9730 EVT MemVT =
LD->getMemoryVT();
9731 if (MemVT != MVT::i32 && MemVT != MVT::i64)
9735 unsigned AS =
LD->getAddressSpace();
9738 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
9740 (MemVT == MVT::i32) ? RTLIB::AEABI_UREAD4 : RTLIB::AEABI_UREAD8;
9748 Opts, dl,
LD->getChain());
9773 EVT MemVT =
ST->getMemoryVT();
9774 if (MemVT != MVT::i32 && MemVT != MVT::i64)
9778 unsigned AS =
ST->getAddressSpace();
9781 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
9783 (MemVT == MVT::i32) ? RTLIB::AEABI_UWRITE4 : RTLIB::AEABI_UWRITE8;
9792 if (
ST->isTruncatingStore())
9797 makeLibCall(DAG, LC, MVT::isVoid, {StoreVal,
ST->getBasePtr()}, Opts,
9798 dl,
ST->getChain());
9800 return CallResult.second;
9811 EVT MemVT = LD->getMemoryVT();
9812 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9813 MemVT == MVT::v16i1) &&
9814 "Expected a predicate type!");
9815 assert(MemVT ==
Op.getValueType());
9817 "Expected a non-extending load");
9818 assert(LD->isUnindexed() &&
"Expected a unindexed load");
9832 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(),
9834 LD->getMemOperand());
9841 if (MemVT != MVT::v16i1)
9850 EVT MemVT =
LD->getMemoryVT();
9852 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
9853 !Subtarget->isThumb1Only() &&
LD->isVolatile() &&
9854 LD->getAlign() >= Subtarget->getDualLoadStoreAlignment()) {
9855 assert(
LD->isUnindexed() &&
"Loads should be unindexed at this point.");
9858 ARMISD::LDRD, dl, DAG.
getVTList({MVT::i32, MVT::i32, MVT::Other}),
9859 {LD->getChain(), LD->getBasePtr()}, MemVT,
LD->getMemOperand());
9864 }
else if (MemVT == MVT::i32 || MemVT == MVT::i64) {
9865 auto Pair = LowerAEABIUnalignedLoad(
SDValue(
N, 0), DAG);
9867 Results.push_back(Pair.first);
9868 Results.push_back(Pair.second);
9875 EVT MemVT = ST->getMemoryVT();
9876 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9877 MemVT == MVT::v16i1) &&
9878 "Expected a predicate type!");
9879 assert(MemVT == ST->getValue().getValueType());
9880 assert(!ST->isTruncatingStore() &&
"Expected a non-extending store");
9881 assert(ST->isUnindexed() &&
"Expected a unindexed store");
9886 SDValue Build = ST->getValue();
9887 if (MemVT != MVT::v16i1) {
9900 SDValue GRP = DAG.
getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Build);
9906 ST->getChain(), dl, GRP, ST->getBasePtr(),
9908 ST->getMemOperand());
9914 EVT MemVT =
ST->getMemoryVT();
9916 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
9917 !Subtarget->isThumb1Only() &&
ST->isVolatile() &&
9918 ST->getAlign() >= Subtarget->getDualLoadStoreAlignment()) {
9919 assert(
ST->isUnindexed() &&
"Stores should be unindexed at this point.");
9920 SDNode *
N =
Op.getNode();
9933 {ST->getChain(), Lo, Hi, ST->getBasePtr()},
9934 MemVT,
ST->getMemOperand());
9935 }
else if (Subtarget->hasMVEIntegerOps() &&
9936 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9937 MemVT == MVT::v16i1))) {
9939 }
else if (MemVT == MVT::i32 || MemVT == MVT::i64) {
9940 return LowerAEABIUnalignedStore(
Op, DAG);
9952 MVT VT =
Op.getSimpleValueType();
9954 SDValue PassThru =
N->getPassThru();
9965 VT, dl,
N->getChain(),
N->getBasePtr(),
N->getOffset(), Mask, ZeroVec,
9966 N->getMemoryVT(),
N->getMemOperand(),
N->getAddressingMode(),
9967 N->getExtensionType(),
N->isExpandingLoad());
9970 PassThru.
getOpcode() == ARMISD::VECTOR_REG_CAST) &&
9972 if (!PassThru.
isUndef() && !PassThruIsCastZero)
9979 if (!ST->hasMVEIntegerOps())
9983 unsigned BaseOpcode = 0;
9984 switch (
Op->getOpcode()) {
10000 unsigned NumActiveLanes = NumElts;
10002 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10003 NumActiveLanes == 2) &&
10004 "Only expected a power 2 vector size");
10008 while (NumActiveLanes > 4) {
10009 unsigned RevOpcode = NumActiveLanes == 16 ? ARMISD::VREV16 : ARMISD::VREV32;
10011 Op0 = DAG.
getNode(BaseOpcode, dl, VT, Op0, Rev);
10012 NumActiveLanes /= 2;
10016 if (NumActiveLanes == 4) {
10026 SDValue Res0 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10027 SDValue Res1 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext2, Ext3,
Op->getFlags());
10028 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res0, Res1,
Op->getFlags());
10034 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10038 if (EltVT !=
Op->getValueType(0))
10045 if (!ST->hasMVEFloatOps())
10052 if (!ST->hasNEON())
10060 unsigned PairwiseIntrinsic = 0;
10061 switch (
Op->getOpcode()) {
10065 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
10068 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
10071 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
10074 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10080 unsigned NumActiveLanes = NumElts;
10082 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10083 NumActiveLanes == 2) &&
10084 "Only expected a power 2 vector size");
10090 VT =
Lo.getValueType();
10092 NumActiveLanes /= 2;
10096 while (NumActiveLanes > 1) {
10098 NumActiveLanes /= 2;
10105 if (EltVT !=
Op.getValueType()) {
10106 unsigned Extend = 0;
10107 switch (
Op->getOpcode()) {
10119 Res = DAG.
getNode(Extend, dl,
Op.getValueType(), Res);
10149 SDLoc dl(V0.getNode());
10154 const SDValue Ops[] = {RegClass, V0, SubReg0,
V1, SubReg1};
10161 auto [VLo, VHi] = DAG.
SplitScalar(
V, dl, MVT::i32, MVT::i32);
10171 assert(
N->getValueType(0) == MVT::i64 &&
10172 "AtomicCmpSwap on types less than 64 should be legal");
10181 ARM::CMP_SWAP_64,
SDLoc(
N),
10182 DAG.
getVTList(MVT::Untyped, MVT::Untyped, MVT::Other),
Ops);
10201 EVT VT =
Op.getValueType();
10210 if (isUnsupportedFloatingType(
LHS.getValueType())) {
10212 Chain, IsSignaling);
10213 if (!
RHS.getNode()) {
10231 ARMcc = DAG.
getConstant(CondCode2, dl, MVT::i32);
10232 Result = getCMOV(dl, VT, Result,
True, ARMcc, Cmp, DAG);
10249 MVT SVT =
Op.getOperand(0).getSimpleValueType();
10252 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
10265 if (!IsSigned && Subtarget->isThumb1Only()) {
10283 Sub1Result, Sub1Result, Flags1);
10298 if (
Op.getValueType() != MVT::i32)
10312 unsigned Opcode = ARMISD::SUBC;
10321 bool CanUseAdd =
false;
10337 Opcode = ARMISD::ADDC;
10361 SDValue Result1 = DAG.
getNode(ARMISD::CMOV, dl, MVT::i32, OpResult, One,
10362 GTCondValue, Flags);
10366 SDValue Result2 = DAG.
getNode(ARMISD::CMOV, dl, MVT::i32, Result1, MinusOne,
10367 LTCondValue, Flags);
10369 if (
Op.getValueType() != MVT::i32)
10377 switch (
Op.getOpcode()) {
10411 case ISD::BITCAST:
return ExpandBITCAST(
Op.getNode(), DAG, Subtarget);
10415 case ISD::SREM:
return LowerREM(
Op.getNode(), DAG);
10416 case ISD::UREM:
return LowerREM(
Op.getNode(), DAG);
10438 return LowerSET_FPMODE(
Op, DAG);
10440 return LowerRESET_FPMODE(
Op, DAG);
10444 !
Op.getValueType().isVector())
10445 return LowerDIV_Windows(
Op, DAG,
true);
10449 !
Op.getValueType().isVector())
10450 return LowerDIV_Windows(
Op, DAG,
false);
10466 return LowerALUO(
Op, DAG);
10474 EVT MemVT = LD->getMemoryVT();
10475 if (Subtarget->hasMVEIntegerOps() &&
10476 (MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10477 MemVT == MVT::v16i1))
10480 auto Pair = LowerAEABIUnalignedLoad(
Op, DAG);
10486 return LowerSTORE(
Op, DAG, Subtarget);
10508 return LowerDYNAMIC_STACKALLOC(
Op, DAG);
10517 return LowerSPONENTRY(
Op, DAG);
10519 return LowerFP_TO_BF16(
Op, DAG);
10520 case ARMISD::WIN__DBZCHK:
return SDValue();
10523 return LowerCMP(
Op, DAG);
10525 return LowerABS(
Op, DAG);
10530 assert((
Op.getOperand(1).getValueType() == MVT::f16 ||
10531 Op.getOperand(1).getValueType() == MVT::bf16) &&
10532 "Expected custom lowering of rounding operations only for f16");
10535 {
Op.getOperand(0),
Op.getOperand(1)});
10536 return DAG.
getNode(
Op.getOpcode(),
DL, {Op.getValueType(), MVT::Other},
10537 {Ext.getValue(1), Ext.getValue(0)});
10544 unsigned IntNo =
N->getConstantOperandVal(0);
10546 if (IntNo == Intrinsic::arm_smlald)
10547 Opc = ARMISD::SMLALD;
10548 else if (IntNo == Intrinsic::arm_smlaldx)
10549 Opc = ARMISD::SMLALDX;
10550 else if (IntNo == Intrinsic::arm_smlsld)
10551 Opc = ARMISD::SMLSLD;
10552 else if (IntNo == Intrinsic::arm_smlsldx)
10553 Opc = ARMISD::SMLSLDX;
10559 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(3), dl, MVT::i32, MVT::i32);
10563 N->getOperand(1),
N->getOperand(2),
10575 switch (
N->getOpcode()) {
10582 Res = ExpandBITCAST(
N, DAG, Subtarget);
10591 Res = LowerREM(
N, DAG);
10595 Res = LowerDivRem(
SDValue(
N, 0), DAG);
10612 "can only expand DIV on Windows");
10624 Res = LowerAEABIUnalignedStore(
SDValue(
N, 0), DAG);
10653 "ROPI/RWPI not currently supported with SjLj");
10666 unsigned PCAdj = (
isThumb || isThumb2) ? 4 : 8;
10672 : &ARM::GPRRegClass;
10690 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10696 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10702 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10720 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10725 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10730 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10735 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
10741 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
10778 const TargetInstrInfo *
TII = Subtarget->getInstrInfo();
10786 : &ARM::GPRnopcRegClass;
10790 DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2>> CallSiteNumToLPad;
10791 unsigned MaxCSNum = 0;
10792 for (MachineBasicBlock &BB : *MF) {
10798 for (MachineInstr &
II : BB) {
10799 if (!
II.isEHLabel())
10802 MCSymbol *Sym =
II.getOperand(0).getMCSymbol();
10803 if (!MF->hasCallSiteLandingPad(Sym))
continue;
10805 SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym);
10806 for (
unsigned Idx : CallSiteIdxs) {
10807 CallSiteNumToLPad[
Idx].push_back(&BB);
10808 MaxCSNum = std::max(MaxCSNum, Idx);
10815 std::vector<MachineBasicBlock*> LPadList;
10816 SmallPtrSet<MachineBasicBlock*, 32> InvokeBBs;
10817 LPadList.reserve(CallSiteNumToLPad.
size());
10818 for (
unsigned I = 1;
I <= MaxCSNum; ++
I) {
10819 SmallVectorImpl<MachineBasicBlock*> &MBBList = CallSiteNumToLPad[
I];
10820 for (MachineBasicBlock *
MBB : MBBList) {
10821 LPadList.push_back(
MBB);
10826 assert(!LPadList.empty() &&
10827 "No landing pad destinations for the dispatch jump table!");
10830 MachineJumpTableInfo *JTI =
10837 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
10840 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
10842 BuildMI(TrapBB, dl,
TII->get(Subtarget->isThumb() ? ARM::tTRAP : ARM::TRAP));
10845 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
10849 MF->insert(MF->end(), DispatchBB);
10850 MF->insert(MF->end(), DispContBB);
10851 MF->insert(MF->end(), TrapBB);
10855 SetupEntryBlockForSjLj(
MI,
MBB, DispatchBB, FI);
10857 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand(
10861 MachineInstrBuilder MIB;
10862 MIB =
BuildMI(DispatchBB, dl,
TII->get(ARM::Int_eh_sjlj_dispatchsetup));
10864 const ARMBaseInstrInfo *AII =
static_cast<const ARMBaseInstrInfo*
>(
TII);
10874 unsigned NumLPads = LPadList.size();
10875 if (Subtarget->isThumb2()) {
10877 BuildMI(DispatchBB, dl,
TII->get(ARM::t2LDRi12), NewVReg1)
10883 if (NumLPads < 256) {
10884 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPri))
10886 .
addImm(LPadList.size())
10890 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVi16), VReg1)
10891 .
addImm(NumLPads & 0xFFFF)
10894 unsigned VReg2 = VReg1;
10895 if ((NumLPads & 0xFFFF0000) != 0) {
10897 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVTi16), VReg2)
10903 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPrr))
10909 BuildMI(DispatchBB, dl,
TII->get(ARM::t2Bcc))
10915 BuildMI(DispContBB, dl,
TII->get(ARM::t2LEApcrelJT), NewVReg3)
10920 BuildMI(DispContBB, dl,
TII->get(ARM::t2ADDrs), NewVReg4)
10927 BuildMI(DispContBB, dl,
TII->get(ARM::t2BR_JT))
10931 }
else if (Subtarget->isThumb()) {
10933 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRspi), NewVReg1)
10939 if (NumLPads < 256) {
10940 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPi8))
10945 MachineConstantPool *
ConstantPool = MF->getConstantPool();
10947 const Constant *
C = ConstantInt::get(Int32Ty, NumLPads);
10950 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
10954 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRpci))
10958 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPr))
10964 BuildMI(DispatchBB, dl,
TII->get(ARM::tBcc))
10970 BuildMI(DispContBB, dl,
TII->get(ARM::tLSLri), NewVReg2)
10977 BuildMI(DispContBB, dl,
TII->get(ARM::tLEApcrelJT), NewVReg3)
10982 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg4)
10988 MachineMemOperand *JTMMOLd =
10993 BuildMI(DispContBB, dl,
TII->get(ARM::tLDRi), NewVReg5)
10999 unsigned NewVReg6 = NewVReg5;
11000 if (IsPositionIndependent) {
11002 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg6)
11009 BuildMI(DispContBB, dl,
TII->get(ARM::tBR_JTr))
11014 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRi12), NewVReg1)
11020 if (NumLPads < 256) {
11021 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPri))
11025 }
else if (Subtarget->hasV6T2Ops() &&
isUInt<16>(NumLPads)) {
11027 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVi16), VReg1)
11028 .
addImm(NumLPads & 0xFFFF)
11031 unsigned VReg2 = VReg1;
11032 if ((NumLPads & 0xFFFF0000) != 0) {
11034 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVTi16), VReg2)
11040 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11045 MachineConstantPool *
ConstantPool = MF->getConstantPool();
11047 const Constant *
C = ConstantInt::get(Int32Ty, NumLPads);
11050 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
11054 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRcp))
11059 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11071 BuildMI(DispContBB, dl,
TII->get(ARM::MOVsi), NewVReg3)
11077 BuildMI(DispContBB, dl,
TII->get(ARM::LEApcrelJT), NewVReg4)
11081 MachineMemOperand *JTMMOLd =
11085 BuildMI(DispContBB, dl,
TII->get(ARM::LDRrs), NewVReg5)
11092 if (IsPositionIndependent) {
11093 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTadd))
11098 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTr))
11105 SmallPtrSet<MachineBasicBlock*, 8> SeenMBBs;
11106 for (MachineBasicBlock *CurMBB : LPadList) {
11107 if (SeenMBBs.
insert(CurMBB).second)
11114 for (MachineBasicBlock *BB : InvokeBBs) {
11118 SmallVector<MachineBasicBlock*, 4> Successors(BB->successors());
11119 while (!Successors.empty()) {
11120 MachineBasicBlock *SMBB = Successors.pop_back_val();
11122 BB->removeSuccessor(SMBB);
11128 BB->normalizeSuccProbs();
11135 II = BB->rbegin(), IE = BB->rend();
II != IE; ++
II) {
11136 if (!
II->isCall())
continue;
11138 DenseSet<unsigned> DefRegs;
11140 OI =
II->operands_begin(), OE =
II->operands_end();
11142 if (!OI->isReg())
continue;
11143 DefRegs.
insert(OI->getReg());
11146 MachineInstrBuilder MIB(*MF, &*
II);
11148 for (
unsigned i = 0; SavedRegs[i] != 0; ++i) {
11149 unsigned Reg = SavedRegs[i];
11150 if (Subtarget->isThumb2() &&
11151 !ARM::tGPRRegClass.contains(
Reg) &&
11152 !ARM::hGPRRegClass.contains(
Reg))
11154 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(
Reg))
11156 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(
Reg))
11168 for (MachineBasicBlock *MBBLPad : MBBLPads)
11169 MBBLPad->setIsEHPad(
false);
11172 MI.eraseFromParent();
11185static unsigned getLdOpcode(
unsigned LdSize,
bool IsThumb1,
bool IsThumb2) {
11187 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11188 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11190 return LdSize == 4 ? ARM::tLDRi
11191 : LdSize == 2 ? ARM::tLDRHi
11192 : LdSize == 1 ? ARM::tLDRBi : 0;
11194 return LdSize == 4 ? ARM::t2LDR_POST
11195 : LdSize == 2 ? ARM::t2LDRH_POST
11196 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11197 return LdSize == 4 ? ARM::LDR_POST_IMM
11198 : LdSize == 2 ? ARM::LDRH_POST
11199 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11204static unsigned getStOpcode(
unsigned StSize,
bool IsThumb1,
bool IsThumb2) {
11206 return StSize == 16 ? ARM::VST1q32wb_fixed
11207 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11209 return StSize == 4 ? ARM::tSTRi
11210 : StSize == 2 ? ARM::tSTRHi
11211 : StSize == 1 ? ARM::tSTRBi : 0;
11213 return StSize == 4 ? ARM::t2STR_POST
11214 : StSize == 2 ? ARM::t2STRH_POST
11215 : StSize == 1 ? ARM::t2STRB_POST : 0;
11216 return StSize == 4 ? ARM::STR_POST_IMM
11217 : StSize == 2 ? ARM::STRH_POST
11218 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11225 unsigned LdSize,
unsigned Data,
unsigned AddrIn,
11226 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11227 unsigned LdOpc =
getLdOpcode(LdSize, IsThumb1, IsThumb2);
11228 assert(LdOpc != 0 &&
"Should have a load opcode");
11235 }
else if (IsThumb1) {
11241 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11246 }
else if (IsThumb2) {
11266 unsigned StSize,
unsigned Data,
unsigned AddrIn,
11267 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11268 unsigned StOpc =
getStOpcode(StSize, IsThumb1, IsThumb2);
11269 assert(StOpc != 0 &&
"Should have a store opcode");
11271 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11276 }
else if (IsThumb1) {
11283 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11288 }
else if (IsThumb2) {
11289 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11295 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11310 const TargetInstrInfo *
TII = Subtarget->getInstrInfo();
11316 unsigned SizeVal =
MI.getOperand(2).getImm();
11322 unsigned UnitSize = 0;
11326 bool IsThumb1 = Subtarget->isThumb1Only();
11327 bool IsThumb2 = Subtarget->isThumb2();
11328 bool IsThumb = Subtarget->isThumb();
11330 if (Alignment & 1) {
11332 }
else if (Alignment & 2) {
11337 Subtarget->hasNEON()) {
11338 if ((Alignment % 16 == 0) && SizeVal >= 16)
11340 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11349 bool IsNeon = UnitSize >= 8;
11350 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11352 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11353 : UnitSize == 8 ? &ARM::DPRRegClass
11356 unsigned BytesLeft = SizeVal % UnitSize;
11357 unsigned LoopSize = SizeVal - BytesLeft;
11359 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11363 unsigned srcIn = src;
11364 unsigned destIn = dest;
11365 for (
unsigned i = 0; i < LoopSize; i+=UnitSize) {
11370 IsThumb1, IsThumb2);
11372 IsThumb1, IsThumb2);
11380 for (
unsigned i = 0; i < BytesLeft; i++) {
11385 IsThumb1, IsThumb2);
11387 IsThumb1, IsThumb2);
11391 MI.eraseFromParent();
11417 MF->
insert(It, loopMBB);
11418 MF->
insert(It, exitMBB);
11421 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
11432 if (Subtarget->useMovt()) {
11433 BuildMI(BB, dl,
TII->get(IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11436 }
else if (Subtarget->genExecuteOnly()) {
11437 assert(IsThumb &&
"Non-thumb expected to have used movt");
11442 const Constant *
C = ConstantInt::get(Int32Ty, LoopSize);
11447 MachineMemOperand *CPMMO =
11471 MachineBasicBlock *entryBB = BB;
11486 BuildMI(BB, dl,
TII->get(ARM::PHI), destPhi)
11494 IsThumb1, IsThumb2);
11496 IsThumb1, IsThumb2);
11500 BuildMI(*BB, BB->
end(), dl,
TII->get(ARM::tSUBi8), varLoop)
11506 MachineInstrBuilder MIB =
11508 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
11517 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11526 auto StartOfExit = exitMBB->
begin();
11530 unsigned srcIn = srcLoop;
11531 unsigned destIn = destLoop;
11532 for (
unsigned i = 0; i < BytesLeft; i++) {
11536 emitPostLd(BB, StartOfExit,
TII, dl, 1, scratch, srcIn, srcOut,
11537 IsThumb1, IsThumb2);
11538 emitPostSt(BB, StartOfExit,
TII, dl, 1, scratch, destIn, destOut,
11539 IsThumb1, IsThumb2);
11544 MI.eraseFromParent();
11552 const TargetInstrInfo &
TII = *Subtarget->getInstrInfo();
11555 assert(TM.getTargetTriple().isOSWindows() &&
11556 "__chkstk is only supported on Windows");
11557 assert(Subtarget->isThumb2() &&
"Windows on ARM requires Thumb-2 mode");
11577 RTLIB::LibcallImpl ChkStkLibcall =
getLibcallImpl(RTLIB::STACK_PROBE);
11578 if (ChkStkLibcall == RTLIB::Unsupported)
11582 switch (TM.getCodeModel()) {
11626 MI.eraseFromParent();
11635 const TargetInstrInfo *
TII = Subtarget->getInstrInfo();
11650 .
addReg(
MI.getOperand(0).getReg())
11658 MI.eraseFromParent();
11682 if (miI == BB->
end()) {
11684 if (Succ->isLiveIn(ARM::CPSR))
11690 SelectItr->addRegisterKilled(ARM::CPSR,
TRI);
11702 Register AddDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11703 BuildMI(TpEntry, Dl,
TII->get(ARM::t2ADDri), AddDestReg)
11709 Register LsrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11710 BuildMI(TpEntry, Dl,
TII->get(ARM::t2LSRri), LsrDestReg)
11716 Register TotalIterationsReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11717 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg)
11720 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopStart))
11721 .
addUse(TotalIterationsReg)
11728 return TotalIterationsReg;
11739 Register TotalIterationsReg,
bool IsMemcpy) {
11746 SrcPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11747 CurrSrcReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11748 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), SrcPhiReg)
11756 Register DestPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11757 Register CurrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11758 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), DestPhiReg)
11765 Register LoopCounterPhiReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11766 Register RemainingLoopIterationsReg =
11767 MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11768 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), LoopCounterPhiReg)
11769 .
addUse(TotalIterationsReg)
11771 .
addUse(RemainingLoopIterationsReg)
11775 Register PredCounterPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11776 Register RemainingElementsReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11777 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), PredCounterPhiReg)
11778 .
addUse(ElementCountReg)
11780 .
addUse(RemainingElementsReg)
11784 Register VccrReg =
MRI.createVirtualRegister(&ARM::VCCRRegClass);
11785 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VCTP8), VccrReg)
11786 .
addUse(PredCounterPhiReg)
11791 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2SUBri), RemainingElementsReg)
11792 .
addUse(PredCounterPhiReg)
11800 SrcValueReg =
MRI.createVirtualRegister(&ARM::MQPRRegClass);
11801 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VLDRBU8_post))
11810 SrcValueReg = OpSrcReg;
11812 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VSTRBU8_post))
11823 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg)
11824 .
addUse(LoopCounterPhiReg)
11827 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopEnd))
11828 .
addUse(RemainingLoopIterationsReg)
11846 "Invalid call instruction for a KCFI check");
11849 switch (
MBBI->getOpcode()) {
11852 case ARM::BLX_pred:
11853 case ARM::BLX_noip:
11854 case ARM::BLX_pred_noip:
11856 TargetOp = &
MBBI->getOperand(0);
11858 case ARM::TCRETURNri:
11859 case ARM::TCRETURNrinotr12:
11860 case ARM::TAILJMPr:
11861 case ARM::TAILJMPr4:
11862 TargetOp = &
MBBI->getOperand(0);
11868 case ARM::tBLXr_noip:
11869 case ARM::tBX_CALL:
11870 TargetOp = &
MBBI->getOperand(2);
11873 case ARM::tTAILJMPr:
11874 TargetOp = &
MBBI->getOperand(0);
11880 assert(TargetOp && TargetOp->
isReg() &&
"Invalid target operand");
11884 unsigned KCFICheckOpcode;
11885 if (Subtarget->isThumb()) {
11886 if (Subtarget->isThumb2()) {
11887 KCFICheckOpcode = ARM::KCFI_CHECK_Thumb2;
11889 KCFICheckOpcode = ARM::KCFI_CHECK_Thumb1;
11892 KCFICheckOpcode = ARM::KCFI_CHECK_ARM;
11906 bool isThumb2 = Subtarget->isThumb2();
11907 switch (
MI.getOpcode()) {
11914 case ARM::tLDR_postidx: {
11918 .
add(
MI.getOperand(2))
11919 .
add(
MI.getOperand(3))
11920 .
add(
MI.getOperand(4))
11921 .
add(
MI.getOperand(0))
11923 MI.eraseFromParent();
11927 case ARM::MVE_MEMCPYLOOPINST:
11928 case ARM::MVE_MEMSETLOOPINST: {
11958 Register OpDestReg =
MI.getOperand(0).getReg();
11959 Register OpSrcReg =
MI.getOperand(1).getReg();
11960 Register OpSizeReg =
MI.getOperand(2).getReg();
11980 if (TpExit == BB) {
11982 "block containing memcpy/memset Pseudo");
11995 bool IsMemcpy =
MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
11997 OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy);
12000 Properties.resetNoPHIs();
12012 MI.eraseFromParent();
12022 case ARM::t2STR_preidx:
12023 MI.setDesc(
TII->get(ARM::t2STR_PRE));
12025 case ARM::t2STRB_preidx:
12026 MI.setDesc(
TII->get(ARM::t2STRB_PRE));
12028 case ARM::t2STRH_preidx:
12029 MI.setDesc(
TII->get(ARM::t2STRH_PRE));
12032 case ARM::STRi_preidx:
12033 case ARM::STRBi_preidx: {
12034 unsigned NewOpc =
MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
12035 : ARM::STRB_PRE_IMM;
12037 unsigned Offset =
MI.getOperand(4).getImm();
12045 .
add(
MI.getOperand(0))
12046 .
add(
MI.getOperand(1))
12047 .
add(
MI.getOperand(2))
12049 .
add(
MI.getOperand(5))
12050 .
add(
MI.getOperand(6))
12052 MI.eraseFromParent();
12055 case ARM::STRr_preidx:
12056 case ARM::STRBr_preidx:
12057 case ARM::STRH_preidx: {
12059 switch (
MI.getOpcode()) {
12061 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG;
break;
12062 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG;
break;
12063 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE;
break;
12068 MI.eraseFromParent();
12072 case ARM::tMOVCCr_pseudo: {
12090 F->insert(It, copy0MBB);
12091 F->insert(It, sinkMBB);
12094 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
12100 if (!
MI.killsRegister(ARM::CPSR,
nullptr) &&
12116 .
addImm(
MI.getOperand(3).getImm())
12117 .
addReg(
MI.getOperand(4).getReg());
12132 .
addReg(
MI.getOperand(1).getReg())
12134 .
addReg(
MI.getOperand(2).getReg())
12137 MI.eraseFromParent();
12142 case ARM::BCCZi64: {
12148 bool RHSisZero =
MI.getOpcode() == ARM::BCCZi64;
12153 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12157 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12163 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12167 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12177 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12186 MI.eraseFromParent();
12190 case ARM::Int_eh_sjlj_setjmp:
12191 case ARM::Int_eh_sjlj_setjmp_nofp:
12192 case ARM::tInt_eh_sjlj_setjmp:
12193 case ARM::t2Int_eh_sjlj_setjmp:
12194 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12197 case ARM::Int_eh_sjlj_setup_dispatch:
12198 EmitSjLjDispatchBlock(
MI, BB);
12200 case ARM::COPY_STRUCT_BYVAL_I32:
12202 return EmitStructByval(
MI, BB);
12203 case ARM::WIN__CHKSTK:
12204 return EmitLowered__chkstk(
MI, BB);
12205 case ARM::WIN__DBZCHK:
12206 return EmitLowered__dbzchk(
MI, BB);
12215 bool isThumb1 =
Subtarget->isThumb1Only();
12222 if (!
Node->hasAnyUseOfValue(0)) {
12223 MI.getOperand(0).setIsDead(
true);
12225 if (!
Node->hasAnyUseOfValue(1)) {
12226 MI.getOperand(1).setIsDead(
true);
12230 for (
unsigned I = 0;
I !=
MI.getOperand(4).
getImm(); ++
I) {
12231 Register TmpReg =
MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
12232 : &ARM::GPRRegClass);
12239 if (
MI.getOpcode() == ARM::MEMCPY) {
12260 MI.getDesc().getNumOperands() + 5 -
MI.getDesc().getSize()
12261 &&
"converted opcode should be the same except for cc_out"
12262 " (and, on Thumb1, pred)");
12270 if (Subtarget->isThumb1Only()) {
12271 for (
unsigned c =
MCID->getNumOperands() - 4; c--;) {
12272 MI.addOperand(
MI.getOperand(1));
12273 MI.removeOperand(1);
12277 for (
unsigned i =
MI.getNumOperands(); i--;) {
12279 if (
op.isReg() &&
op.isUse()) {
12282 MI.tieOperands(DefIdx, i);
12290 ccOutIdx =
MCID->getNumOperands() - 1;
12292 ccOutIdx =
MCID->getNumOperands() - 1;
12296 if (!
MI.hasOptionalDef() || !
MCID->operands()[ccOutIdx].isOptionalDef()) {
12297 assert(!NewOpc &&
"Optional cc_out operand required");
12303 bool deadCPSR =
false;
12304 for (
unsigned i =
MCID->getNumOperands(), e =
MI.getNumOperands(); i != e;
12311 MI.removeOperand(i);
12316 assert(!NewOpc &&
"Optional cc_out operand required");
12319 assert(deadCPSR == !
Node->hasAnyUseOfValue(1) &&
"inconsistent dead flag");
12321 assert(!
MI.getOperand(ccOutIdx).getReg() &&
12322 "expect uninitialized optional cc_out operand");
12324 if (!Subtarget->isThumb1Only())
12361 switch (
N->getOpcode()) {
12362 default:
return false;
12364 CC =
N->getOperand(0);
12386 EVT VT =
N->getValueType(0);
12387 CC =
N->getOperand(0);
12388 if (
CC.getValueType() != MVT::i1 ||
CC.getOpcode() !=
ISD::SETCC)
12434 EVT VT =
N->getValueType(0);
12437 bool SwapSelectOps;
12439 NonConstantVal, DAG))
12445 OtherOp, NonConstantVal);
12451 CCOp, TrueVal, FalseVal);
12471 if (
N->getOpcode() == ARMISD::VUZP)
12475 if (
N->getOpcode() == ARMISD::VTRN &&
N->getValueType(0) == MVT::v2i32)
12490 if (!
N->getValueType(0).is64BitVector())
12498 EVT VT =
N->getValueType(0);
12537 EVT VT =
N->getValueType(0);
12543 Opcode = Intrinsic::arm_neon_vpaddls;
12545 Opcode = Intrinsic::arm_neon_vpaddlu;
12573 EVT VT =
N->getValueType(0);
12588 unsigned nextIndex = 0;
12610 if (!
C0 || !
C1 ||
C0->getZExtValue() != nextIndex
12611 ||
C1->getZExtValue() != nextIndex+1)
12639 Ops.push_back(Vec);
12656 return DAG.
getNode(ExtOp, dl, VT, tmp);
12687 if (SRA.getOpcode() !=
ISD::SRA) {
12694 if (Const->getZExtValue() != 31)
12699 if (SRA.getOperand(0) !=
Mul)
12703 SDLoc dl(AddcNode);
12704 unsigned Opcode = 0;
12709 Opcode = ARMISD::SMLALBB;
12710 Op0 =
Mul.getOperand(0);
12711 Op1 =
Mul.getOperand(1);
12713 Opcode = ARMISD::SMLALBT;
12714 Op0 =
Mul.getOperand(0);
12715 Op1 =
Mul.getOperand(1).getOperand(0);
12717 Opcode = ARMISD::SMLALTB;
12718 Op0 =
Mul.getOperand(0).getOperand(0);
12719 Op1 =
Mul.getOperand(1);
12721 Opcode = ARMISD::SMLALTT;
12722 Op0 =
Mul->getOperand(0).getOperand(0);
12723 Op1 =
Mul->getOperand(1).getOperand(0);
12767 AddeSubeNode->
getOpcode() == ARMISD::SUBE) &&
12768 "Expect an ADDE or SUBE");
12772 "ADDE node has the wrong inputs");
12779 if ((AddeSubeNode->
getOpcode() == ARMISD::ADDE &&
12780 AddcSubcNode->
getOpcode() != ARMISD::ADDC) ||
12781 (AddeSubeNode->
getOpcode() == ARMISD::SUBE &&
12782 AddcSubcNode->
getOpcode() != ARMISD::SUBC))
12794 "Expect ADDC with two result values. First: i32");
12798 if (AddeSubeNode->
getOpcode() == ARMISD::ADDE &&
12814 bool IsLeftOperandMUL =
false;
12819 IsLeftOperandMUL =
true;
12830 SDValue *LowAddSub =
nullptr;
12833 if ((AddeSubeOp0 != MULOp.
getValue(1)) && (AddeSubeOp1 != MULOp.
getValue(1)))
12836 if (IsLeftOperandMUL)
12837 HiAddSub = &AddeSubeOp1;
12839 HiAddSub = &AddeSubeOp0;
12844 if (AddcSubcOp0 == MULOp.
getValue(0)) {
12845 LoMul = &AddcSubcOp0;
12846 LowAddSub = &AddcSubcOp1;
12848 if (AddcSubcOp1 == MULOp.
getValue(0)) {
12849 LoMul = &AddcSubcOp1;
12850 LowAddSub = &AddcSubcOp0;
12858 if (AddcSubcNode == HiAddSub->getNode() ||
12875 FinalOpc == ARMISD::SMLAL &&
12879 Ops.push_back(*HiAddSub);
12880 if (AddcSubcNode->
getOpcode() == ARMISD::SUBC) {
12881 FinalOpc = ARMISD::SMMLSR;
12883 FinalOpc = ARMISD::SMMLAR;
12888 return SDValue(AddeSubeNode, 0);
12889 }
else if (AddcSubcNode->
getOpcode() == ARMISD::SUBC)
12895 Ops.push_back(*LowAddSub);
12896 Ops.push_back(*HiAddSub);
12909 return SDValue(AddeSubeNode, 0);
12929 if (AddcNode->
getOpcode() != ARMISD::ADDC)
12933 SDNode *UmlalNode =
nullptr;
12977 SDNode* AddcNode =
N->getOperand(2).getNode();
12978 SDNode* AddeNode =
N->getOperand(3).getNode();
12979 if ((AddcNode->
getOpcode() == ARMISD::ADDC) &&
12980 (AddeNode->
getOpcode() == ARMISD::ADDE) &&
12986 {N->getOperand(0), N->getOperand(1),
12987 AddcNode->getOperand(0), AddcNode->getOperand(1)});
12997 if (
N->getOpcode() == ARMISD::SUBC &&
N->hasAnyUseOfValue(1)) {
13001 if (
LHS->getOpcode() == ARMISD::ADDE &&
13011 int32_t imm =
C->getSExtValue();
13012 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13015 unsigned Opcode = (
N->getOpcode() == ARMISD::ADDC) ? ARMISD::SUBC
13017 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
N->getOperand(0),
RHS);
13032 int64_t imm =
C->getSExtValue();
13041 unsigned Opcode = (
N->getOpcode() == ARMISD::ADDE) ? ARMISD::SUBE
13043 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
13044 N->getOperand(0),
RHS,
N->getOperand(2));
13069 SetCC =
N->getOperand(0);
13073 TrueVal =
N->getOperand(1);
13074 FalseVal =
N->getOperand(2);
13076 LHS =
N->getOperand(0);
13077 RHS =
N->getOperand(1);
13078 CC =
N->getOperandCondCode(4);
13079 TrueVal =
N->getOperand(2);
13080 FalseVal =
N->getOperand(3);
13085 unsigned int Opcode = 0;
13089 Opcode = ARMISD::VMINVu;
13095 Opcode = ARMISD::VMINVs;
13101 Opcode = ARMISD::VMAXVu;
13107 Opcode = ARMISD::VMAXVs;
13114 switch (TrueVal->getOpcode()) {
13133 if (TrueVal !=
LHS || FalseVal !=
RHS)
13136 EVT LeftType =
LHS->getValueType(0);
13137 EVT RightType =
RHS->getValueType(0);
13140 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13144 if (VectorScalarType != MVT::i32)
13152 if (VectorScalarType != MVT::i32)
13165 EVT VT =
N->getValueType(0);
13173 Shft =
N->getOperand(0);
13180 Cmp.getOperand(0) !=
N->getOperand(1) ||
13181 Cmp.getOperand(1) !=
N->getOperand(2))
13183 Shft =
N->getOperand(1);
13195 ScalarType = MVT::i8;
13198 case (1 << 15) - 1:
13199 ScalarType = MVT::i16;
13202 case (1ULL << 31) - 1:
13203 ScalarType = MVT::i32;
13234 unsigned LegalLanes = 128 / (ShftAmt + 1);
13246 Inp0 = DAG.
getNode(ARMISD::VECTOR_REG_CAST,
DL, LegalVecVT, Inp0);
13247 Inp1 = DAG.
getNode(ARMISD::VECTOR_REG_CAST,
DL, LegalVecVT, Inp1);
13248 SDValue VQDMULH = DAG.
getNode(ARMISD::VQDMULH,
DL, LegalVecVT, Inp0, Inp1);
13249 SDValue Trunc = DAG.
getNode(ARMISD::VECTOR_REG_CAST,
DL, ExtVecVT, VQDMULH);
13265 SDValue VQDMULH = DAG.
getNode(ARMISD::VQDMULH,
DL, LegalVecVT, Inp0, Inp1);
13280 if (
N->getOperand(0).getOpcode() == ARMISD::PREDICATE_CAST &&
13282 unsigned C =
N->getOperand(0).getConstantOperandVal(0);
13284 return N->getOperand(2);
13286 return N->getOperand(1);
13301 if (
N->getOperand(0).getOpcode() !=
ISD::XOR)
13311 if (!Const || !Const->isOne())
13329 EVT VT =
N->getValueType(0);
13360 Opc = Intrinsic::arm_mve_vctp32;
13363 Opc = Intrinsic::arm_mve_vctp16;
13366 Opc = Intrinsic::arm_mve_vctp8;
13420 EVT VT =
N->getValueType(0);
13426 switch (
Op.getOpcode()) {
13428 case ARMISD::VADDVs:
13429 case ARMISD::VADDVu:
13430 case ARMISD::VMLAVs:
13431 case ARMISD::VMLAVu:
13451 unsigned N0RedOp = 0;
13458 unsigned N1RedOp = 0;
13472 if (
SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13474 if (
SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13481 auto DistrubuteVecReduceLoad = [&](
SDValue N0,
SDValue N1,
bool IsForward) {
13505 if (!BaseLocDecomp0.getBase() ||
13506 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13507 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13509 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13511 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13521 if (IsBefore < 0) {
13524 }
else if (IsBefore > 0) {
13537 }
else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13547 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13557 if (
SDValue R = DistrubuteVecReduceLoad(N0, N1,
true))
13559 if (
SDValue R = DistrubuteVecReduceLoad(N1, N0,
false))
13572 EVT VT =
N->getValueType(0);
13577 if (VT != MVT::i64)
13588 auto MakeVecReduce = [&](
unsigned Opcode,
unsigned OpcodeA,
SDValue NA,
13608 unsigned S = VecRed->
getOpcode() == OpcodeA ? 2 : 0;
13617 if (
SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N0, N1))
13619 if (
SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N0, N1))
13621 if (
SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N1, N0))
13623 if (
SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N1, N0))
13625 if (
SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N0, N1))
13627 if (
SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N0, N1))
13629 if (
SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N1, N0))
13631 if (
SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N1, N0))
13633 if (
SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N0, N1))
13635 if (
SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N0, N1))
13637 if (
SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N1, N0))
13639 if (
SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N1, N0))
13641 if (
SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N0, N1))
13643 if (
SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N0, N1))
13645 if (
SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N1, N0))
13647 if (
SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N1, N0))
13657 "Expected shift op");
13659 SDValue ShiftLHS =
N->getOperand(0);
13673 if (Subtarget->isThumb1Only()) {
13684 if (Const->getAPIntValue().ult(256))
13687 Const->getAPIntValue().sgt(-256))
13703 (
N->getOperand(0).getOpcode() ==
ISD::SHL ||
13704 N->getOperand(0).getOpcode() ==
ISD::SRL) &&
13705 "Expected XOR(SHIFT) pattern");
13710 if (XorC && ShiftC) {
13711 unsigned MaskIdx, MaskLen;
13712 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13713 unsigned ShiftAmt = ShiftC->getZExtValue();
13714 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
13715 if (
N->getOperand(0).getOpcode() ==
ISD::SHL)
13716 return MaskIdx == ShiftAmt && MaskLen == (
BitWidth - ShiftAmt);
13717 return MaskIdx == 0 && MaskLen == (
BitWidth - ShiftAmt);
13727 N->getOperand(0).getOpcode() ==
ISD::SRL) ||
13729 N->getOperand(0).getOpcode() ==
ISD::SHL)) &&
13730 "Expected shift-shift mask");
13732 if (!Subtarget->isThumb1Only())
13735 EVT VT =
N->getValueType(0);
13743 unsigned BinOpcode,
EVT VT,
unsigned SelectOpcode,
SDValue X,
13745 return Subtarget->hasMVEIntegerOps() &&
isTypeLegal(VT) &&
13750 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps()) {
13751 if (Subtarget->isThumb1Only())
13765 return Subtarget->hasVFP2Base();
13767 return Subtarget->hasVFP2Base();
13769 return Subtarget->hasFP64();
13772 return Subtarget->hasMVEFloatOps();
13801 if (ST->isThumb1Only())
13805 for (
auto *U :
N->users()) {
13806 switch(U->getOpcode()) {
13824 if (U->getOperand(0).getOpcode() ==
ISD::SHL ||
13825 U->getOperand(1).getOpcode() ==
ISD::SHL)
13835 if (
N->getOperand(0).getOpcode() !=
ISD::SHL)
13842 if (!C1ShlC2 || !C2)
13845 APInt C2Int = C2->getAPIntValue();
13846 APInt C1Int = C1ShlC2->getAPIntValue();
13848 if (C2Int.
uge(C2Width))
13854 if ((C1Int & Mask) != C1Int)
13861 auto LargeImm = [](
const APInt &
Imm) {
13862 unsigned Zeros =
Imm.countl_zero() +
Imm.countr_zero();
13863 return Imm.getBitWidth() - Zeros > 8;
13866 if (LargeImm(C1Int) || LargeImm(C2Int))
13878 SHL.dump();
N->dump());
13939 if (
Op.hasOneUse() && ShiftAmt &&
13940 ShiftAmt->
getZExtValue() ==
Op.getValueType().getScalarSizeInBits() - 1)
13998 if (!
Subtarget->hasMVEIntegerOps() || !
N->getValueType(0).isVector())
14019 return DCI.
DAG.
getNode(ARMISD::VDUP, dl,
N->getValueType(0), Negate);
14059 EVT VT =
N->getValueType(0);
14070 EVT VT =
N->getValueType(0);
14071 if (VT != MVT::v2i64)
14082 return Op->getOperand(0);
14096 And =
And->getOperand(0);
14101 Mask = Mask->getOperand(0);
14104 Mask.getValueType() != MVT::v4i32)
14110 return And->getOperand(0);
14115 if (
SDValue Op0 = IsSignExt(N0)) {
14116 if (
SDValue Op1 = IsSignExt(N1)) {
14117 SDValue New0a = DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
14118 SDValue New1a = DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
14119 return DAG.
getNode(ARMISD::VMULLs, dl, VT, New0a, New1a);
14122 if (
SDValue Op0 = IsZeroExt(N0)) {
14123 if (
SDValue Op1 = IsZeroExt(N1)) {
14124 SDValue New0a = DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
14125 SDValue New1a = DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
14126 return DAG.
getNode(ARMISD::VMULLu, dl, VT, New0a, New1a);
14138 EVT VT =
N->getValueType(0);
14139 if (
Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14150 if (VT != MVT::i32)
14157 int64_t MulAmt =
C->getSExtValue();
14160 ShiftAmt = ShiftAmt & (32 - 1);
14165 MulAmt >>= ShiftAmt;
14226 if (
N->getValueType(0) != MVT::i32)
14235 if (
C1 == 255 ||
C1 == 65535)
14238 SDNode *N0 =
N->getOperand(0).getNode();
14252 if (!C2 || C2 >= 32)
14296 if (Trailing == C2 && C2 + C3 < 32) {
14309 if (Leading == C2 && C2 + C3 < 32) {
14337 EVT VT =
N->getValueType(0);
14341 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14344 APInt SplatBits, SplatUndef;
14345 unsigned SplatBitSize;
14348 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14349 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14350 SplatBitSize == 64) {
14357 DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, VbicVT,
N->getOperand(0));
14359 return DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vbic);
14389 SDValue SRL = OR->getOperand(0);
14390 SDValue SHL = OR->getOperand(1);
14393 SRL = OR->getOperand(1);
14394 SHL = OR->getOperand(0);
14401 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) ||
14405 SDNode *SMULLOHI = SRL.getOperand(0).getNode();
14406 if (SRL.getOperand(0) !=
SDValue(SMULLOHI, 0) ||
14407 SHL.getOperand(0) !=
SDValue(SMULLOHI, 1))
14426 unsigned Opcode = 0;
14427 if (
isS16(OpS16, DAG))
14428 Opcode = ARMISD::SMULWB;
14430 Opcode = ARMISD::SMULWT;
14448 EVT VT =
N->getValueType(0);
14463 if (VT != MVT::i32)
14476 if (Mask == 0xffff)
14483 if ((Val & ~Mask) != Val)
14489 Res = DAG.
getNode(ARMISD::BFI,
DL, VT, N00,
14508 (Mask == ~Mask2)) {
14512 (Mask == 0xffff || Mask == 0xffff0000))
14518 Res = DAG.
getNode(ARMISD::BFI,
DL, VT, N00, Res,
14525 (~Mask == Mask2)) {
14529 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14585 if (
N->getOpcode() == ARMISD::VCMP)
14587 else if (
N->getOpcode() == ARMISD::VCMPZ)
14602 EVT VT =
N->getValueType(0);
14607 auto IsFreelyInvertable = [&](
SDValue V) {
14608 if (
V->getOpcode() == ARMISD::VCMP ||
V->getOpcode() == ARMISD::VCMPZ)
14614 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14632 if (AndOp.getOpcode() !=
ISD::AND)
14636 SDValue Mask = AndOp.getOperand(1);
14646 bool IsShiftRight =
false;
14649 if (ShiftOp.
getOpcode() == ARMISD::VSHRuIMM) {
14650 IsShiftRight =
true;
14653 }
else if (ShiftOp.
getOpcode() == ARMISD::VSHLIMM) {
14661 APInt RequiredMask = IsShiftRight
14664 if (MaskBits != RequiredMask)
14667 unsigned Opc = IsShiftRight ? ARMISD::VSRIIMM : ARMISD::VSLIIMM;
14677 EVT VT =
N->getValueType(0);
14683 if (
Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14684 VT == MVT::v8i1 || VT == MVT::v16i1))
14687 APInt SplatBits, SplatUndef;
14688 unsigned SplatBitSize;
14691 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14692 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14693 SplatBitSize == 64) {
14700 DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, VorrVT,
N->getOperand(0));
14702 return DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vorr);
14723 (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32)))) {
14726 return ShiftInsert;
14730 return ShiftInsert;
14744 unsigned SplatBitSize;
14747 APInt SplatBits0, SplatBits1;
14751 if (BVN0 && BVN0->
isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
14752 HasAnyUndefs) && !HasAnyUndefs) {
14753 if (BVN1 && BVN1->
isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
14754 HasAnyUndefs) && !HasAnyUndefs) {
14759 SplatBits0 == ~SplatBits1) {
14770 DAG.
getNode(ARMISD::VBSP, dl, CanonicalVT, Mask,
LHS,
RHS);
14771 return DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Result);
14791 if (CSINC.
getOpcode() != ARMISD::CSINC)
14793 if (CSINC.
getOpcode() == ARMISD::CSINC &&
14806 EVT VT =
N->getValueType(0);
14826 if (
TLI->isConstTrueVal(N1) &&
14849 assert(
N->getOpcode() == ARMISD::BFI);
14852 ToMask =
~N->getConstantOperandAPInt(2);
14872 unsigned LastActiveBitInA =
A.countr_zero();
14873 unsigned FirstActiveBitInB =
B.getBitWidth() -
B.countl_zero() - 1;
14874 return LastActiveBitInA - 1 == FirstActiveBitInB;
14879 APInt ToMask, FromMask;
14884 if (
V.getOpcode() != ARMISD::BFI)
14887 APInt NewToMask, NewFromMask;
14889 if (NewFrom != From)
14893 if ((NewToMask & ToMask).getBoolValue())
14918 unsigned InvMask =
N->getConstantOperandVal(2);
14922 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14923 "undefined behavior");
14924 unsigned Mask = (1u << Width) - 1;
14926 if ((Mask & (~Mask2)) == 0)
14928 N->getOperand(0), N1.
getOperand(0),
N->getOperand(2));
14935 APInt ToMask1, FromMask1;
14938 APInt ToMask2, FromMask2;
14944 APInt NewFromMask = FromMask1 | FromMask2;
14945 APInt NewToMask = ToMask1 | ToMask2;
14947 EVT VT =
N->getValueType(0);
14950 if (NewFromMask[0] == 0)
14953 return DAG.
getNode(ARMISD::BFI, dl, VT, CombineBFI.getOperand(0), From1,
14961 if (
N->getOperand(0).getOpcode() == ARMISD::BFI) {
14962 APInt ToMask1 =
~N->getConstantOperandAPInt(2);
14963 APInt ToMask2 = ~N0.getConstantOperandAPInt(2);
14965 if (!N0.
hasOneUse() || (ToMask1 & ToMask2) != 0 ||
14969 EVT VT =
N->getValueType(0);
14972 N->getOperand(1),
N->getOperand(2));
14984 if (Cmp->getOpcode() != ARMISD::CMPZ || !
isNullConstant(Cmp->getOperand(1)))
14986 SDValue CSInc = Cmp->getOperand(0);
14996 if (CSInc.
getOpcode() == ARMISD::CSINC &&
15036 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ)
15037 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15040 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
15042 N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15055 SDValue InDouble =
N->getOperand(0);
15070 SDValue BasePtr = LD->getBasePtr();
15072 DAG.
getLoad(MVT::i32,
DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
15073 LD->getAlign(), LD->getMemOperand()->getFlags());
15079 LD->getPointerInfo().getWithOffset(4),
15081 LD->getMemOperand()->getFlags());
15100 BV.
getOpcode() == ARMISD::VECTOR_REG_CAST) &&
15151 if (Op0.
getOpcode() == ARMISD::VMOVRRD &&
15164 if (Op0->
getOpcode() == ARMISD::VMOVrh)
15177 if (Copy.getValueType() == MVT::f32 &&
15179 bool HasGlue = Copy->getNumOperands() == 3;
15180 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1),
15181 HasGlue ? Copy->getOperand(2) :
SDValue()};
15182 EVT OutTys[] = {
N->getValueType(0), MVT::Other, MVT::Glue};
15201 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15202 LN0->getMemoryVT() == MVT::i16) {
15205 LN0->getBasePtr(), LN0->getMemOperand());
15215 if (
TLI.SimplifyDemandedBits(Op0, DemandedMask, DCI))
15223 EVT VT =
N->getValueType(0);
15254 if (Op0->
getOpcode() == ARMISD::VMOVsr)
15264 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
15265 for (
unsigned i = 0; i < NumElts; ++i) {
15266 SDNode *Elt =
N->getOperand(i).getNode();
15283 if (
N->getNumOperands() == 2)
15289 EVT VT =
N->getValueType(0);
15295 for (
unsigned i = 0; i < NumElts; ++i) {
15314 unsigned NumParts =
N->getNumOperands();
15327 unsigned EltBits =
SubVecTy.getScalarSizeInBits();
15328 if (EltBits != 8 && EltBits != 16 && EltBits != 32)
15332 for (
const SDValue &Operand :
N->op_values()) {
15342 unsigned Bytes =
SubVecTy.getStoreSize().getFixedValue();
15362 static constexpr Intrinsic::ID NEONLoads[] = {Intrinsic::arm_neon_vld2,
15363 Intrinsic::arm_neon_vld3,
15364 Intrinsic::arm_neon_vld4};
15368 IID =
NumParts == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
15404 unsigned NumParts =
N->getNumOperands();
15418 unsigned EltBits =
SubVecTy.getScalarSizeInBits();
15419 if (EltBits != 8 && EltBits != 16 && EltBits != 32)
15424 unsigned MinNumElements =
SubVecTy.getVectorMinNumElements();
15445 if (
Subtarget->hasMVEIntegerOps() &&
Load->getAlign() < EltBits / 8)
15448 static constexpr Intrinsic::ID NEONLoads[] = {Intrinsic::arm_neon_vld2,
15449 Intrinsic::arm_neon_vld3,
15450 Intrinsic::arm_neon_vld4};
15457 IID =
NumParts == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
15465 Load->getMemoryVT(),
Load->getMemOperand());
15468 for (
unsigned Idx = 0; Idx <
NumParts; Idx++)
15480 EVT VT =
N->getValueType(0);
15486 auto IsExtractFromBuildVec = [](
SDValue V,
SDValue &BV, int64_t &Lane) {
15490 BV = BV.getOperand(0);
15492 BV.getNumOperands() == 4 &&
15493 BV.getValueType().getScalarSizeInBits() == 32;
15496 int64_t Lane1, Lane2;
15497 if (IsExtractFromBuildVec(
N->getOperand(0), BV1, Lane1) &&
15498 IsExtractFromBuildVec(
N->getOperand(1), BV2, Lane2))
15527 assert(EltVT == MVT::f32 &&
"Unexpected type!");
15532 Use->getValueType(0).isFloatingPoint())
15540 unsigned NumOfBitCastedElts = 0;
15542 unsigned NumOfRelevantElts = NumElts;
15543 for (
unsigned Idx = 0; Idx < NumElts; ++Idx) {
15548 ++NumOfBitCastedElts;
15552 --NumOfRelevantElts;
15556 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15563 if (!
TLI.isTypeLegal(VecVT))
15573 for (
unsigned Idx = 0 ; Idx < NumElts; ++Idx) {
15578 V->getOperand(0).getValueType() == MVT::i32)
15580 V =
V.getOperand(0);
15597 EVT VT =
N->getValueType(0);
15602 if (
Op->getOpcode() == ARMISD::PREDICATE_CAST) {
15604 if (
Op->getOperand(0).getValueType() == VT)
15605 return Op->getOperand(0);
15606 return DCI.
DAG.
getNode(ARMISD::PREDICATE_CAST, dl, VT,
Op->getOperand(0));
15613 DCI.
DAG.
getNode(ARMISD::PREDICATE_CAST, dl, VT,
Op->getOperand(0));
15620 if (
Op.getValueType() == MVT::i32) {
15623 if (
TLI.SimplifyDemandedBits(
Op, DemandedMask, DCI))
15631 EVT VT =
N->getValueType(0);
15636 if (ST->isLittle())
15640 if (
Op.getValueType() == VT)
15647 if (
Op->getOpcode() == ARMISD::VECTOR_REG_CAST) {
15649 if (
Op->getOperand(0).getValueType() == VT)
15650 return Op->getOperand(0);
15651 return DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, VT,
Op->getOperand(0));
15662 EVT VT =
N->getValueType(0);
15670 return DAG.
getNode(ARMISD::VCMPZ, dl, VT, Op0,
N->getOperand(2));
15676 return DAG.
getNode(ARMISD::VCMPZ, dl, VT, Op1,
15680 return DAG.
getNode(ARMISD::VCMP, dl, VT, Op1, Op0,
15693 EVT VT =
N->getValueType(0);
15694 SDNode *Elt =
N->getOperand(1).getNode();
15709 Vec,
V,
N->getOperand(2));
15719 EVT VT =
N->getValueType(0);
15747 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15748 isa<ConstantSDNode>(V->getOperand(1)) &&
15749 V->getConstantOperandVal(1) == Lane + 1 &&
15750 V->getOperand(0).getResNo() == ResNo;
15752 if (OtherIt == Op0->
users().
end())
15757 SDValue OtherExt(*OtherIt, 0);
15769 DCI.
DAG.
getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v2f64, Op0),
15772 DCI.
DAG.
getNode(ARMISD::VMOVRRD, dl, {MVT::i32, MVT::i32},
F64);
15782 EVT VT =
N->getValueType(0);
15786 if (Op0->
getOpcode() == ARMISD::VDUP) {
15788 if (VT == MVT::f16 &&
X.getValueType() == MVT::i32)
15789 return DCI.
DAG.
getNode(ARMISD::VMOVhr, dl, VT,
X);
15790 if (VT == MVT::i32 &&
X.getValueType() == MVT::f16)
15791 return DCI.
DAG.
getNode(ARMISD::VMOVrh, dl, VT,
X);
15792 if (VT == MVT::f32 &&
X.getValueType() == MVT::i32)
15795 while (
X.getValueType() != VT &&
X->getOpcode() ==
ISD::BITCAST)
15796 X =
X->getOperand(0);
15797 if (
X.getValueType() == VT)
15805 return Op0.
getOperand(
N->getConstantOperandVal(1));
15815 unsigned Offset =
N->getConstantOperandVal(1);
15817 if (MOV.
getOpcode() == ARMISD::VMOVDRR)
15827 unsigned Idx =
N->getConstantOperandVal(1);
15842 unsigned Lane =
N->getConstantOperandVal(1);
15870 EVT VT =
N->getValueType(0);
15873 if (
Op.getOpcode() == ARMISD::VGETLANEu &&
15875 Op.getOperand(0).getValueType().getScalarType())
15876 return DAG.
getNode(ARMISD::VGETLANEs,
SDLoc(
N), VT,
Op.getOperand(0),
15885 SDValue SubVec =
N->getOperand(1);
15886 uint64_t IdxVal =
N->getConstantOperandVal(2);
15897 if (IdxVal == 0 && Vec.
isUndef())
15903 (IdxVal != 0 && IdxVal != NumSubElts))
15934 ARMISD::VMOVN,
DL, VT,
15940 ARMISD::VMOVN,
DL, VT,
15976 EVT VT =
N->getValueType(0);
15977 if (!
TLI.isTypeLegal(VT) ||
15987 unsigned HalfElts = NumElts/2;
15989 for (
unsigned n = 0; n < NumElts; ++n) {
15992 if (MaskElt < (
int)HalfElts)
15994 else if (MaskElt >= (
int)NumElts && MaskElt < (
int)(NumElts + HalfElts))
15995 NewElt = HalfElts + MaskElt - NumElts;
16029 const unsigned MaxSteps = 1024;
16038 bool SimpleConstIncOnly,
16046 bool isLoadOp =
true;
16047 bool isLaneOp =
false;
16050 bool hasAlignment =
true;
16051 unsigned NewOpc = 0;
16052 unsigned NumVecs = 0;
16053 if (
Target.isIntrinsic) {
16054 unsigned IntNo =
N->getConstantOperandVal(1);
16058 case Intrinsic::arm_neon_vld1:
16062 case Intrinsic::arm_neon_vld2:
16066 case Intrinsic::arm_neon_vld3:
16070 case Intrinsic::arm_neon_vld4:
16074 case Intrinsic::arm_neon_vld1x2:
16077 hasAlignment =
false;
16079 case Intrinsic::arm_neon_vld1x3:
16082 hasAlignment =
false;
16084 case Intrinsic::arm_neon_vld1x4:
16087 hasAlignment =
false;
16089 case Intrinsic::arm_neon_vld2dup:
16093 case Intrinsic::arm_neon_vld3dup:
16097 case Intrinsic::arm_neon_vld4dup:
16101 case Intrinsic::arm_neon_vld2lane:
16106 case Intrinsic::arm_neon_vld3lane:
16111 case Intrinsic::arm_neon_vld4lane:
16116 case Intrinsic::arm_neon_vst1:
16121 case Intrinsic::arm_neon_vst2:
16122 NewOpc = ARMISD::VST2_UPD;
16126 case Intrinsic::arm_neon_vst3:
16131 case Intrinsic::arm_neon_vst4:
16132 NewOpc = ARMISD::VST4_UPD;
16136 case Intrinsic::arm_neon_vst2lane:
16142 case Intrinsic::arm_neon_vst3lane:
16148 case Intrinsic::arm_neon_vst4lane:
16154 case Intrinsic::arm_neon_vst1x2:
16158 hasAlignment =
false;
16160 case Intrinsic::arm_neon_vst1x3:
16164 hasAlignment =
false;
16166 case Intrinsic::arm_neon_vst1x4:
16170 hasAlignment =
false;
16175 switch (
N->getOpcode()) {
16211 VecTy =
N->getValueType(0);
16212 }
else if (
Target.isIntrinsic) {
16213 VecTy =
N->getOperand(
Target.AddrOpIdx + 1).getValueType();
16216 "Node has to be a load, a store, or an intrinsic!");
16217 VecTy =
N->getOperand(1).getValueType();
16225 if (isLaneOp || isVLDDUPOp)
16228 if (NumBytes >= 3 * 16 &&
User.ConstInc != NumBytes) {
16234 if (SimpleConstIncOnly &&
User.ConstInc != NumBytes)
16243 EVT AlignedVecTy = VecTy;
16263 assert(NumVecs == 1 &&
"Unexpected multi-element generic load/store.");
16264 assert(!isLaneOp &&
"Unexpected generic load/store lane.");
16275 Alignment =
Align(1);
16281 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16283 for (n = 0; n < NumResultVecs; ++n)
16284 Tys[n] = AlignedVecTy;
16285 Tys[n++] = MVT::i32;
16286 Tys[n] = MVT::Other;
16291 Ops.push_back(
N->getOperand(0));
16292 Ops.push_back(
N->getOperand(
Target.AddrOpIdx));
16297 Ops.push_back(StN->getValue());
16301 unsigned LastOperand =
16302 hasAlignment ?
N->getNumOperands() - 1 :
N->getNumOperands();
16303 for (
unsigned i =
Target.AddrOpIdx + 1; i < LastOperand; ++i)
16304 Ops.push_back(
N->getOperand(i));
16308 Ops.push_back(DAG.
getConstant(Alignment.value(), dl, MVT::i32));
16312 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::STORE) {
16319 MemN->getMemOperand());
16323 for (
unsigned i = 0; i < NumResultVecs; ++i)
16328 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::LOAD) {
16329 SDValue &LdVal = NewResults[0];
16365 switch (
N->getOpcode()) {
16369 *Ptr =
N->getOperand(0);
16370 *CInc =
N->getOperand(1);
16377 *Ptr =
N->getOperand(1);
16378 *CInc =
N->getOperand(2);
16405 SDValue Addr =
N->getOperand(AddrOpIdx);
16416 unsigned ConstInc =
16421 if (BaseUpdates.
size() >= MaxBaseUpdates)
16442 unsigned UserOffset =
16445 if (!UserOffset || UserOffset <=
Offset)
16448 unsigned NewConstInc = UserOffset -
Offset;
16451 if (BaseUpdates.
size() >= MaxBaseUpdates)
16459 unsigned NumValidUpd = BaseUpdates.
size();
16460 for (
unsigned I = 0;
I < NumValidUpd;
I++) {
16471 return LHS.ConstInc <
RHS.ConstInc;
16500 unsigned IntNo =
N->getConstantOperandVal(1);
16501 if (IntNo == Intrinsic::arm_mve_vst2q &&
N->getConstantOperandVal(5) != 1)
16503 if (IntNo == Intrinsic::arm_mve_vst4q &&
N->getConstantOperandVal(7) != 3)
16520 const unsigned MaxSteps = 1024;
16526 bool isLoadOp =
true;
16527 unsigned NewOpc = 0;
16528 unsigned NumVecs = 0;
16532 case Intrinsic::arm_mve_vld2q:
16536 case Intrinsic::arm_mve_vld4q:
16540 case Intrinsic::arm_mve_vst2q:
16541 NewOpc = ARMISD::VST2_UPD;
16545 case Intrinsic::arm_mve_vst4q:
16546 NewOpc = ARMISD::VST4_UPD;
16555 VecTy =
N->getValueType(0);
16557 VecTy =
N->getOperand(3).getValueType();
16571 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16573 for (n = 0; n < NumResultVecs; ++n)
16575 Tys[n++] = MVT::i32;
16576 Tys[n] = MVT::Other;
16581 Ops.push_back(
N->getOperand(0));
16582 Ops.push_back(
N->getOperand(2));
16583 Ops.push_back(Inc);
16585 for (
unsigned i = 3; i <
N->getNumOperands(); ++i)
16586 Ops.push_back(
N->getOperand(i));
16589 MemN->getMemOperand());
16593 for (
unsigned i = 0; i < NumResultVecs; ++i)
16612 EVT VT =
N->getValueType(0);
16618 SDNode *VLD =
N->getOperand(0).getNode();
16621 unsigned NumVecs = 0;
16622 unsigned NewOpc = 0;
16624 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16627 }
else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16630 }
else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16642 if (
Use.getResNo() == NumVecs)
16645 if (
User->getOpcode() != ARMISD::VDUPLANE ||
16646 VLDLaneNo !=
User->getConstantOperandVal(1))
16653 for (n = 0; n < NumVecs; ++n)
16655 Tys[n] = MVT::Other;
16665 unsigned ResNo =
Use.getResNo();
16667 if (ResNo == NumVecs)
16674 std::vector<SDValue> VLDDupResults;
16675 for (
unsigned n = 0; n < NumVecs; ++n)
16689 EVT VT =
N->getValueType(0);
16696 ExtractVT = MVT::i32;
16698 N->getOperand(0),
N->getOperand(1));
16710 Op =
Op.getOperand(0);
16711 if (
Op.getOpcode() != ARMISD::VMOVIMM &&
Op.getOpcode() != ARMISD::VMVNIMM)
16715 unsigned EltSize =
Op.getScalarValueSizeInBits();
16717 unsigned Imm =
Op.getConstantOperandVal(0);
16736 if (
Op.getValueType() == MVT::f32)
16737 return DAG.
getNode(ARMISD::VDUP, dl,
N->getValueType(0),
16739 else if (
Op.getValueType() == MVT::f16)
16740 return DAG.
getNode(ARMISD::VDUP, dl,
N->getValueType(0),
16741 DAG.
getNode(ARMISD::VMOVrh, dl, MVT::i32,
Op));
16751 if (LD &&
Op.hasOneUse() && LD->isUnindexed() &&
16752 LD->getMemoryVT() ==
N->getValueType(0).getVectorElementType()) {
16753 SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1),
16758 LD->getMemoryVT(), LD->getMemOperand());
16769 EVT VT =
N->getValueType(0);
16788 SDValue StVal = St->getValue();
16790 if (!St->isTruncatingStore() || !VT.
isVector())
16793 EVT StVT = St->getMemoryVT();
16795 assert(StVT != VT &&
"Cannot truncate to the same type");
16805 if (0 != (NumElems * FromEltSz) % ToEltSz)
16808 unsigned SizeRatio = FromEltSz / ToEltSz;
16813 NumElems * SizeRatio);
16819 for (
unsigned i = 0; i < NumElems; ++i)
16824 if (!
TLI.isTypeLegal(WideVecVT))
16833 MVT StoreType = MVT::i8;
16835 if (
TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16839 if (!
TLI.isTypeLegal(StoreType))
16851 SDValue BasePtr = St->getBasePtr();
16855 for (
unsigned I = 0;
I <
E;
I++) {
16859 DAG.
getStore(St->getChain(),
DL, SubVec, BasePtr, St->getPointerInfo(),
16860 St->getAlign(), St->getMemOperand()->getFlags());
16873 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16875 SDValue Trunc = St->getValue();
16886 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16889 unsigned NumElements = 4;
16906 unsigned Off0 = Rev ? NumElts : 0;
16907 unsigned Off1 = Rev ? 0 : NumElts;
16909 for (
unsigned I = 0;
I < NumElts;
I += 2) {
16910 if (M[
I] >= 0 && M[
I] != (
int)(Off0 +
I / 2))
16912 if (M[
I + 1] >= 0 && M[
I + 1] != (
int)(Off1 +
I / 2))
16920 if (isVMOVNShuffle(Shuffle,
false) || isVMOVNShuffle(Shuffle,
true))
16927 SDValue BasePtr = St->getBasePtr();
16928 Align Alignment = St->getBaseAlign();
16940 unsigned NewOffset = i * NumElements * ToEltVT.
getSizeInBits() / 8;
16951 Extract = DAG.
getNode(ARMISD::VECTOR_REG_CAST,
DL, MVT::v4i32, FPTrunc);
16954 Ch,
DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset),
16955 NewToVT, Alignment, MMOFlags, AAInfo);
16966 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16968 SDValue Trunc = St->getValue();
16978 SDValue BasePtr = St->getBasePtr();
16979 Align Alignment = St->getBaseAlign();
16988 unsigned NewOffset =
16995 Ch,
DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset),
16996 NewToVT, Alignment, MMOFlags, AAInfo);
17007 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
17009 SDValue Extract = St->getValue();
17018 {Extract.getOperand(0), Extract.getOperand(1)});
17026 SDValue BasePtr = St->getBasePtr();
17027 Align Alignment = St->getBaseAlign();
17032 St->getPointerInfo(), NewToVT, Alignment,
17051 SDValue StoredValue = ST->getValue();
17069 unsigned EltBits =
SubVecTy.getScalarSizeInBits();
17070 if (EltBits != 8 && EltBits != 16 && EltBits != 32)
17075 unsigned ResNo =
Use.getResNo();
17077 if (Stores[ResNo] || !
Store)
17079 Stores[ResNo] =
Store;
17088 unsigned Bytes =
SubVecTy.getStoreSize().getFixedValue();
17094 static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::arm_neon_vst2,
17095 Intrinsic::arm_neon_vst3,
17096 Intrinsic::arm_neon_vst4};
17105 IID =
NumParts == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
17125 for (
unsigned Stage = 0; Stage <
NumParts; Stage++) {
17129 Ops.front() = NewStore;
17145 unsigned NumParts =
N->getNumOperands();
17149 SDNode *InterleaveOp =
N->getOperand(0).getNode();
17151 InterleaveOp->getNumOperands() !=
NumParts)
17155 if (
N->getOperand(
I) !=
SDValue(InterleaveOp,
I))
17158 Ops.append(InterleaveOp->op_begin(), InterleaveOp->op_end());
17177 SDValue WideValue = ST->getValue();
17180 ValueInterleaveOps))
17193 EVT SubVecTy = ValueInterleaveOps[0].getValueType();
17199 unsigned EltBits =
SubVecTy.getScalarSizeInBits();
17200 if (EltBits != 8 && EltBits != 16 && EltBits != 32)
17202 if (
Subtarget->hasMVEIntegerOps() && ST->getAlign() < EltBits / 8)
17204 static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::arm_neon_vst2,
17205 Intrinsic::arm_neon_vst3,
17206 Intrinsic::arm_neon_vst4};
17209 Ops.push_back(ST->getChain());
17214 IID =
NumParts == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
17218 Ops.push_back(ST->getBasePtr());
17219 Ops.append(ValueInterleaveOps);
17225 ST->getMemoryVT(), ST->getMemOperand());
17228 for (
unsigned Stage = 0; Stage <
NumParts; Stage++) {
17232 ST->getMemoryVT(), ST->getMemOperand());
17233 Ops.front() = NewStore;
17244 if (St->isVolatile())
17246 SDValue StVal = St->getValue();
17277 SDValue BasePtr = St->getBasePtr();
17280 BasePtr, St->getPointerInfo(), St->getBaseAlign(),
17281 St->getMemOperand()->getFlags());
17287 St->getPointerInfo().getWithOffset(4),
17288 St->getBaseAlign(), St->getMemOperand()->
getFlags());
17310 return DAG.
getStore(St->getChain(), dl,
V, St->getBasePtr(),
17311 St->getPointerInfo(), St->getAlign(),
17312 St->getMemOperand()->getFlags(), St->getAAInfo());
17338 if (!
Op.getValueType().isVector() || !
Op.getValueType().isSimple() ||
17346 MVT FloatTy =
Op.getSimpleValueType().getVectorElementType();
17348 MVT IntTy =
N->getSimpleValueType(0).getVectorElementType();
17349 uint32_t IntBits = IntTy.getSizeInBits();
17350 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
17351 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17362 if (
C == -1 ||
C == 0 ||
C > 32)
17367 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
17368 Intrinsic::arm_neon_vcvtfp2fxu;
17371 DAG.
getConstant(IntrinsicOpcode, dl, MVT::i32),
Op->getOperand(0),
17374 if (IntBits < FloatBits)
17390 EVT VT =
N->getValueType(0);
17395 auto isIdentitySplat = [&](
SDValue Op,
bool NSZ) {
17397 Op.getOperand(0).getOpcode() != ARMISD::VMOVIMM)
17399 uint64_t ImmVal =
Op.getOperand(0).getConstantOperandVal(0);
17400 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
17402 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
17415 if (!isIdentitySplat(Op1.
getOperand(2), NSZ))
17426 EVT VT =
N->getValueType(0);
17429 if (!
N->getFlags().hasAllowReassociation())
17436 unsigned Opc =
A.getConstantOperandVal(0);
17437 if (
Opc != Intrinsic::arm_mve_vcmlaq)
17442 A.getOperand(3),
A.getOperand(4));
17478 unsigned OpOpcode =
Op.getNode()->getOpcode();
17479 if (!
N->getValueType(0).isVector() || !
N->getValueType(0).isSimple() ||
17483 SDValue ConstVec =
N->getOperand(1);
17487 MVT FloatTy =
N->getSimpleValueType(0).getVectorElementType();
17489 MVT IntTy =
Op.getOperand(0).getSimpleValueType().getVectorElementType();
17490 uint32_t IntBits = IntTy.getSizeInBits();
17491 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
17492 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17512 int32_t
C = IntVal.exactLogBase2();
17513 if (
C == -1 ||
C == 0 ||
C > 32)
17519 if (IntBits < FloatBits)
17521 NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, ConvInput);
17523 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfxs2fp
17524 : Intrinsic::arm_neon_vcvtfxu2fp;
17532 if (!ST->hasMVEIntegerOps())
17536 EVT ResVT =
N->getValueType(0);
17564 EVT AVT =
A.getValueType();
17570 auto ExtendIfNeeded = [&](
SDValue A,
unsigned ExtendCode) {
17571 EVT AVT =
A.getValueType();
17581 auto IsVADDV = [&](
MVT RetTy,
unsigned ExtendCode,
ArrayRef<MVT> ExtTypes) {
17582 if (ResVT != RetTy || N0->
getOpcode() != ExtendCode)
17585 if (ExtTypeMatches(
A, ExtTypes))
17586 return ExtendIfNeeded(
A, ExtendCode);
17589 auto IsPredVADDV = [&](
MVT RetTy,
unsigned ExtendCode,
ArrayRef<MVT> ExtTypes,
17599 if (ExtTypeMatches(
A, ExtTypes))
17600 return ExtendIfNeeded(
A, ExtendCode);
17603 auto IsVMLAV = [&](
MVT RetTy,
unsigned ExtendCode,
ArrayRef<MVT> ExtTypes,
17613 if (ResVT != RetTy)
17616 if (
Mul->getOpcode() == ExtendCode &&
17617 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17619 Mul =
Mul->getOperand(0);
17628 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17629 A = ExtendIfNeeded(
A, ExtendCode);
17630 B = ExtendIfNeeded(
B, ExtendCode);
17635 auto IsPredVMLAV = [&](
MVT RetTy,
unsigned ExtendCode,
ArrayRef<MVT> ExtTypes,
17648 if (
Mul->getOpcode() == ExtendCode &&
17649 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17651 Mul =
Mul->getOperand(0);
17660 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17661 A = ExtendIfNeeded(
A, ExtendCode);
17662 B = ExtendIfNeeded(
B, ExtendCode);
17673 EVT VT =
Ops[0].getValueType();
17674 if (VT == MVT::v16i8) {
17675 assert((Opcode == ARMISD::VMLALVs || Opcode == ARMISD::VMLALVu) &&
17676 "Unexpected illegal long reduction opcode");
17677 bool IsUnsigned = Opcode == ARMISD::VMLALVu;
17689 DAG.
getNode(IsUnsigned ? ARMISD::VMLALVAu : ARMISD::VMLALVAs, dl,
17702 return DAG.
getNode(ARMISD::VMLAVs, dl, ResVT,
A,
B);
17704 return DAG.
getNode(ARMISD::VMLAVu, dl, ResVT,
A,
B);
17705 if (IsVMLAV(MVT::i64,
ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17707 return Create64bitNode(ARMISD::VMLALVs, {
A,
B});
17708 if (IsVMLAV(MVT::i64,
ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17710 return Create64bitNode(ARMISD::VMLALVu, {
A,
B});
17713 DAG.
getNode(ARMISD::VMLAVs, dl, MVT::i32,
A,
B));
17716 DAG.
getNode(ARMISD::VMLAVu, dl, MVT::i32,
A,
B));
17720 return DAG.
getNode(ARMISD::VMLAVps, dl, ResVT,
A,
B, Mask);
17723 return DAG.
getNode(ARMISD::VMLAVpu, dl, ResVT,
A,
B, Mask);
17726 return Create64bitNode(ARMISD::VMLALVps, {
A,
B, Mask});
17729 return Create64bitNode(ARMISD::VMLALVpu, {
A,
B, Mask});
17732 DAG.
getNode(ARMISD::VMLAVps, dl, MVT::i32,
A,
B, Mask));
17735 DAG.
getNode(ARMISD::VMLAVpu, dl, MVT::i32,
A,
B, Mask));
17738 return DAG.
getNode(ARMISD::VADDVs, dl, ResVT,
A);
17740 return DAG.
getNode(ARMISD::VADDVu, dl, ResVT,
A);
17742 return Create64bitNode(ARMISD::VADDLVs, {
A});
17744 return Create64bitNode(ARMISD::VADDLVu, {
A});
17747 DAG.
getNode(ARMISD::VADDVs, dl, MVT::i32,
A));
17750 DAG.
getNode(ARMISD::VADDVu, dl, MVT::i32,
A));
17753 return DAG.
getNode(ARMISD::VADDVps, dl, ResVT,
A, Mask);
17755 return DAG.
getNode(ARMISD::VADDVpu, dl, ResVT,
A, Mask);
17757 return Create64bitNode(ARMISD::VADDLVps, {
A, Mask});
17759 return Create64bitNode(ARMISD::VADDLVpu, {
A, Mask});
17762 DAG.
getNode(ARMISD::VADDVps, dl, MVT::i32,
A, Mask));
17765 DAG.
getNode(ARMISD::VADDVpu, dl, MVT::i32,
A, Mask));
17772 Op =
Op->getOperand(1);
17774 Op->getOperand(0)->getOpcode() ==
ISD::MUL) {
17776 if (
Mul->getOperand(0) ==
Mul->getOperand(1) &&
17793 unsigned VecOp =
N->getOperand(0).getValueType().isVector() ? 0 : 2;
17795 if (!Shuf || !Shuf->getOperand(1).isUndef())
17800 APInt SetElts(Mask.size(), 0);
17801 for (
int E : Mask) {
17809 if (
N->getNumOperands() != VecOp + 1) {
17811 if (!Shuf2 || !Shuf2->getOperand(1).isUndef() || Shuf2->getMask() != Mask)
17817 if (
Op.getValueType().isVector())
17818 Ops.push_back(
Op.getOperand(0));
17829 unsigned IsTop =
N->getConstantOperandVal(2);
17836 if (Op0->
isUndef() && !IsTop)
17841 if ((Op1->
getOpcode() == ARMISD::VQMOVNs ||
17842 Op1->
getOpcode() == ARMISD::VQMOVNu) &&
17850 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17852 APInt Op0DemandedElts =
17853 IsTop ? Op1DemandedElts
17857 if (
TLI.SimplifyDemandedVectorElts(Op0, Op0DemandedElts, DCI))
17859 if (
TLI.SimplifyDemandedVectorElts(Op1, Op1DemandedElts, DCI))
17868 unsigned IsTop =
N->getConstantOperandVal(2);
17870 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17871 APInt Op0DemandedElts =
17876 if (
TLI.SimplifyDemandedVectorElts(Op0, Op0DemandedElts, DCI))
17883 EVT VT =
N->getValueType(0);
17890 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
17891 LHS.getOperand(1).isUndef() &&
RHS.getOperand(1).isUndef() &&
17895 LHS.getOperand(0),
RHS.getOperand(0));
17910 int ShiftAmt =
C->getSExtValue();
17911 if (ShiftAmt == 0) {
17917 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17918 unsigned NewOpcode =
17919 N->getOpcode() == ARMISD::LSLL ? ARMISD::LSRL : ARMISD::LSLL;
17934 unsigned IntNo =
N->getConstantOperandVal(0);
17945 case Intrinsic::arm_neon_vshifts:
17946 case Intrinsic::arm_neon_vshiftu:
17947 case Intrinsic::arm_neon_vrshifts:
17948 case Intrinsic::arm_neon_vrshiftu:
17949 case Intrinsic::arm_neon_vrshiftn:
17950 case Intrinsic::arm_neon_vqshifts:
17951 case Intrinsic::arm_neon_vqshiftu:
17952 case Intrinsic::arm_neon_vqshiftsu:
17953 case Intrinsic::arm_neon_vqshiftns:
17954 case Intrinsic::arm_neon_vqshiftnu:
17955 case Intrinsic::arm_neon_vqshiftnsu:
17956 case Intrinsic::arm_neon_vqrshiftns:
17957 case Intrinsic::arm_neon_vqrshiftnu:
17958 case Intrinsic::arm_neon_vqrshiftnsu: {
17959 EVT VT =
N->getOperand(1).getValueType();
17961 unsigned VShiftOpc = 0;
17964 case Intrinsic::arm_neon_vshifts:
17965 case Intrinsic::arm_neon_vshiftu:
17967 VShiftOpc = ARMISD::VSHLIMM;
17970 if (
isVShiftRImm(
N->getOperand(2), VT,
false,
true, Cnt)) {
17971 VShiftOpc = (IntNo == Intrinsic::arm_neon_vshifts ? ARMISD::VSHRsIMM
17972 : ARMISD::VSHRuIMM);
17977 case Intrinsic::arm_neon_vrshifts:
17978 case Intrinsic::arm_neon_vrshiftu:
17983 case Intrinsic::arm_neon_vqshifts:
17984 case Intrinsic::arm_neon_vqshiftu:
17989 case Intrinsic::arm_neon_vqshiftsu:
17994 case Intrinsic::arm_neon_vrshiftn:
17995 case Intrinsic::arm_neon_vqshiftns:
17996 case Intrinsic::arm_neon_vqshiftnu:
17997 case Intrinsic::arm_neon_vqshiftnsu:
17998 case Intrinsic::arm_neon_vqrshiftns:
17999 case Intrinsic::arm_neon_vqrshiftnu:
18000 case Intrinsic::arm_neon_vqrshiftnsu:
18012 case Intrinsic::arm_neon_vshifts:
18013 case Intrinsic::arm_neon_vshiftu:
18016 case Intrinsic::arm_neon_vrshifts:
18017 VShiftOpc = ARMISD::VRSHRsIMM;
18019 case Intrinsic::arm_neon_vrshiftu:
18020 VShiftOpc = ARMISD::VRSHRuIMM;
18022 case Intrinsic::arm_neon_vrshiftn:
18023 VShiftOpc = ARMISD::VRSHRNIMM;
18025 case Intrinsic::arm_neon_vqshifts:
18026 VShiftOpc = ARMISD::VQSHLsIMM;
18028 case Intrinsic::arm_neon_vqshiftu:
18029 VShiftOpc = ARMISD::VQSHLuIMM;
18031 case Intrinsic::arm_neon_vqshiftsu:
18032 VShiftOpc = ARMISD::VQSHLsuIMM;
18034 case Intrinsic::arm_neon_vqshiftns:
18035 VShiftOpc = ARMISD::VQSHRNsIMM;
18037 case Intrinsic::arm_neon_vqshiftnu:
18038 VShiftOpc = ARMISD::VQSHRNuIMM;
18040 case Intrinsic::arm_neon_vqshiftnsu:
18041 VShiftOpc = ARMISD::VQSHRNsuIMM;
18043 case Intrinsic::arm_neon_vqrshiftns:
18044 VShiftOpc = ARMISD::VQRSHRNsIMM;
18046 case Intrinsic::arm_neon_vqrshiftnu:
18047 VShiftOpc = ARMISD::VQRSHRNuIMM;
18049 case Intrinsic::arm_neon_vqrshiftnsu:
18050 VShiftOpc = ARMISD::VQRSHRNsuIMM;
18055 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
18056 N->getOperand(1), DAG.
getConstant(Cnt, dl, MVT::i32));
18059 case Intrinsic::arm_neon_vshiftins: {
18060 EVT VT =
N->getOperand(1).getValueType();
18062 unsigned VShiftOpc = 0;
18065 VShiftOpc = ARMISD::VSLIIMM;
18066 else if (
isVShiftRImm(
N->getOperand(3), VT,
false,
true, Cnt))
18067 VShiftOpc = ARMISD::VSRIIMM;
18073 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
18074 N->getOperand(1),
N->getOperand(2),
18078 case Intrinsic::arm_neon_vqrshifts:
18079 case Intrinsic::arm_neon_vqrshiftu:
18083 case Intrinsic::arm_neon_vbsl: {
18085 return DAG.
getNode(ARMISD::VBSP, dl,
N->getValueType(0),
N->getOperand(1),
18086 N->getOperand(2),
N->getOperand(3));
18088 case Intrinsic::arm_mve_vqdmlah:
18089 case Intrinsic::arm_mve_vqdmlash:
18090 case Intrinsic::arm_mve_vqrdmlah:
18091 case Intrinsic::arm_mve_vqrdmlash:
18092 case Intrinsic::arm_mve_vmla_n_predicated:
18093 case Intrinsic::arm_mve_vmlas_n_predicated:
18094 case Intrinsic::arm_mve_vqdmlah_predicated:
18095 case Intrinsic::arm_mve_vqdmlash_predicated:
18096 case Intrinsic::arm_mve_vqrdmlah_predicated:
18097 case Intrinsic::arm_mve_vqrdmlash_predicated: {
18102 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
18109 case Intrinsic::arm_mve_minv:
18110 case Intrinsic::arm_mve_maxv:
18111 case Intrinsic::arm_mve_minav:
18112 case Intrinsic::arm_mve_maxav:
18113 case Intrinsic::arm_mve_minv_predicated:
18114 case Intrinsic::arm_mve_maxv_predicated:
18115 case Intrinsic::arm_mve_minav_predicated:
18116 case Intrinsic::arm_mve_maxav_predicated: {
18119 unsigned BitWidth =
N->getOperand(2)->getValueType(0).getScalarSizeInBits();
18126 case Intrinsic::arm_mve_addv: {
18129 bool Unsigned =
N->getConstantOperandVal(2);
18130 unsigned Opc =
Unsigned ? ARMISD::VADDVu : ARMISD::VADDVs;
18134 case Intrinsic::arm_mve_addlv:
18135 case Intrinsic::arm_mve_addlv_predicated: {
18138 bool Unsigned =
N->getConstantOperandVal(2);
18139 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
18140 (
Unsigned ? ARMISD::VADDLVu : ARMISD::VADDLVs) :
18141 (
Unsigned ? ARMISD::VADDLVpu : ARMISD::VADDLVps);
18144 for (
unsigned i = 1, e =
N->getNumOperands(); i < e; i++)
18146 Ops.push_back(
N->getOperand(i));
18159 EVT VT =
Y.getValueType();
18162 if (Subtarget->hasMVEIntegerOps())
18164 if (Subtarget->hasNEON())
18178 EVT VT =
N->getValueType(0);
18180 if (ST->isThumb1Only() &&
N->getOpcode() ==
ISD::SHL && VT == MVT::i32 &&
18181 N->getOperand(0)->getOpcode() ==
ISD::AND &&
18182 N->getOperand(0)->hasOneUse()) {
18199 if (AndMask == 255 || AndMask == 65535)
18203 if (MaskedBits > ShiftAmt) {
18218 if (ST->hasMVEIntegerOps())
18223 switch (
N->getOpcode()) {
18229 return DAG.
getNode(ARMISD::VSHLIMM, dl, VT,
N->getOperand(0),
18236 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
18237 unsigned VShiftOpc =
18238 (
N->getOpcode() ==
ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
18240 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
18256 if (!LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed() ||
18259 EVT FromVT = LD->getValueType(0);
18260 EVT ToVT =
N->getValueType(0);
18267 unsigned NumElements = 0;
18268 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
18270 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
18272 if (NumElements == 0 ||
18282 SDValue BasePtr = LD->getBasePtr();
18283 Align Alignment = LD->getBaseAlign();
18304 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
18305 Alignment, MMOFlags, AAInfo);
18311 if (FromEltVT == MVT::f16) {
18314 for (
unsigned i = 0; i < Loads.
size(); i++) {
18316 DAG.
getNode(ARMISD::VECTOR_REG_CAST,
DL, MVT::v8f16, Loads[i]);
18335 EVT VT =
N->getValueType(0);
18342 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
18349 if (VT == MVT::i32 &&
18350 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
18355 switch (
N->getOpcode()) {
18358 Opc = ARMISD::VGETLANEs;
18362 Opc = ARMISD::VGETLANEu;
18369 if (ST->hasMVEIntegerOps())
18387 Ops.push_back(Ext);
18397 if (ST->hasMVEFloatOps())
18412 EVT VT =
Op.getValueType();
18415 if (VT != MVT::i32 ||
18431 APInt MaxC = Max.getConstantOperandAPInt(1);
18432 if (MaxC.
sgt(MinC))
18439 if ((MinC + 1).isPowerOf2()) {
18461 APInt Width = MinC - MaxC + 1;
18464 unsigned SatBit = Width.
logBase2() - 1;
18482 EVT VT =
N->getValueType(0);
18485 if (VT == MVT::i32)
18488 if (!ST->hasMVEIntegerOps())
18494 if (VT != MVT::v4i32 && VT != MVT::v8i16)
18497 auto IsSignedSaturate = [&](
SDNode *Min,
SDNode *Max) {
18505 if (VT == MVT::v4i32)
18506 SaturateC =
APInt(32, (1 << 15) - 1,
true);
18508 SaturateC =
APInt(16, (1 << 7) - 1,
true);
18515 MaxC != ~SaturateC)
18520 if (IsSignedSaturate(
N, N0.
getNode())) {
18523 if (VT == MVT::v4i32) {
18524 HalfVT = MVT::v8i16;
18525 ExtVT = MVT::v4i16;
18527 HalfVT = MVT::v16i8;
18542 auto IsUnsignedSaturate = [&](
SDNode *Min) {
18548 if (VT == MVT::v4i32)
18549 SaturateC =
APInt(32, (1 << 16) - 1,
true);
18551 SaturateC =
APInt(16, (1 << 8) - 1,
true);
18560 if (IsUnsignedSaturate(
N)) {
18564 if (VT == MVT::v4i32) {
18565 HalfVT = MVT::v8i16;
18566 ExtConst = 0x0000FFFF;
18568 HalfVT = MVT::v16i8;
18590 const APInt *CV = &
C->getAPIntValue();
18647 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18654 if ((OrCI &
Known.Zero) != OrCI)
18660 EVT VT =
X.getValueType();
18661 unsigned BitInX = AndC->
logBase2();
18669 for (
unsigned BitInY = 0, NumActiveBits = OrCI.
getActiveBits();
18670 BitInY < NumActiveBits; ++BitInY) {
18671 if (OrCI[BitInY] == 0)
18674 Mask.setBit(BitInY);
18675 V = DAG.
getNode(ARMISD::BFI, dl, VT, V,
X,
18691 switch (
N->getOpcode()) {
18706 if (Const->isZero())
18708 else if (Const->isOne())
18712 CC =
N.getOperandCondCode(2);
18716 unsigned IntOp =
N.getConstantOperandVal(1);
18717 if (IntOp != Intrinsic::test_start_loop_iterations &&
18718 IntOp != Intrinsic::loop_decrement_reg)
18744 bool Negate =
false;
18750 Cond =
N->getOperand(1);
18751 Dest =
N->getOperand(2);
18754 CC =
N->getOperandCondCode(1);
18755 Cond =
N->getOperand(2);
18756 Dest =
N->getOperand(4);
18758 if (!Const->isOne() && !Const->isZero())
18760 Imm = Const->getZExtValue();
18789 "unsupported condition");
18794 unsigned IntOp =
Int->getConstantOperandVal(1);
18795 assert((
N->hasOneUse() &&
N->user_begin()->getOpcode() ==
ISD::BR) &&
18796 "expected single br user");
18797 SDNode *Br = *
N->user_begin();
18807 if (IntOp == Intrinsic::test_start_loop_iterations) {
18809 SDValue Setup = DAG.
getNode(ARMISD::WLSSETUP, dl, MVT::i32, Elements);
18811 if (IsTrueIfZero(
CC,
Imm)) {
18813 Res = DAG.
getNode(ARMISD::WLS, dl, MVT::Other,
Ops);
18817 UpdateUncondBr(Br, Dest, DAG);
18819 SDValue Ops[] = {Chain, Setup, OtherTarget};
18820 Res = DAG.
getNode(ARMISD::WLS, dl, MVT::Other,
Ops);
18832 DAG.
getVTList(MVT::i32, MVT::Other), Args);
18840 if (
Target == OtherTarget)
18841 UpdateUncondBr(Br, Dest, DAG);
18847 return DAG.
getNode(ARMISD::LE, dl, MVT::Other, EndArgs);
18856 if (Cmp.getOpcode() != ARMISD::CMPZ)
18861 SDValue LHS = Cmp.getOperand(0);
18862 SDValue RHS = Cmp.getOperand(1);
18871 LHS->getOperand(0)->getOpcode() == ARMISD::CMOV &&
18872 LHS->getOperand(0)->hasOneUse() &&
18876 return DAG.
getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, BB,
18888 EVT VT =
N->getValueType(0);
18889 SDValue FalseVal =
N->getOperand(0);
18890 SDValue TrueVal =
N->getOperand(1);
18898 matchCSET(Opcode, InvertCond, TrueVal, FalseVal, Subtarget)) {
18905 return DAG.
getNode(Opcode, dl, VT, CSetOp, CSetOp, ARMcc, Cmp);
18908 if (Cmp.getOpcode() != ARMISD::CMPZ)
18912 SDValue LHS = Cmp.getOperand(0);
18913 SDValue RHS = Cmp.getOperand(1);
18917 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) {
18941 if (
CC ==
ARMCC::NE && FalseVal == RHS && FalseVal != LHS) {
18942 Res = DAG.
getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc, Cmp);
18946 Res = DAG.
getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc, NewCmp);
18951 if (
CC ==
ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse() &&
18954 return DAG.
getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal,
18955 LHS->getOperand(2), LHS->getOperand(3));
18965 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ ||
18969 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
18971 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
18980 if (!Subtarget->isThumb1Only() && Subtarget->hasV5TOps()) {
18985 if (VT == MVT::i32 && Subtarget->isThumb2() &&
isNullConstant(RHS) &&
19019 Res = DAG.
getNode(ARMISD::CMOV, dl, VT,
Sub, TrueVal, ARMcc,
19031 Res = DAG.
getNode(ARMISD::CMOV, dl, VT,
Sub, FalseVal,
19051 const APInt *TrueConst;
19052 if (Subtarget->isThumb1Only() &&
CC ==
ARMCC::NE &&
19053 ((FalseVal.getOpcode() == ARMISD::SUBC && FalseVal.getOperand(0) == LHS &&
19054 FalseVal.getOperand(1) == RHS) ||
19058 unsigned ShiftAmount = TrueConst->
logBase2();
19073 if (
Known.Zero == 0xfffffffe)
19076 else if (
Known.Zero == 0xffffff00)
19079 else if (
Known.Zero == 0xffff0000)
19092 EVT DstVT =
N->getValueType(0);
19095 if (ST->hasMVEIntegerOps() && Src.getOpcode() == ARMISD::VDUP) {
19096 EVT SrcVT = Src.getValueType();
19098 return DAG.
getNode(ARMISD::VDUP,
SDLoc(
N), DstVT, Src.getOperand(0));
19103 if (Src.getOpcode() == ARMISD::VECTOR_REG_CAST &&
19104 Src.getOperand(0).getValueType().getScalarSizeInBits() <=
19105 Src.getValueType().getScalarSizeInBits())
19106 Src = Src.getOperand(0);
19110 EVT SrcVT = Src.getValueType();
19111 if ((Src.getOpcode() == ARMISD::VMOVIMM ||
19112 Src.getOpcode() == ARMISD::VMVNIMM ||
19113 Src.getOpcode() == ARMISD::VMOVFPIMM) &&
19116 return DAG.
getNode(ARMISD::VECTOR_REG_CAST,
SDLoc(
N), DstVT, Src);
19130 EVT VT =
N->getValueType(0);
19138 if (
N->getNumOperands() == 2 &&
19142 N->getOperand(0).getOperand(1),
19143 N->getOperand(1).getOperand(0),
19144 N->getOperand(1).getOperand(1));
19147 if (
N->getNumOperands() == 2 &&
19153 if (S0->getOperand(0) ==
S1->getOperand(0) &&
19154 S0->getOperand(1) ==
S1->getOperand(1)) {
19157 Mask.append(
S1->getMask().begin(),
S1->getMask().end());
19161 ARMISD::VMOVN,
DL, VT,
19162 DAG.
getNode(ARMISD::VECTOR_REG_CAST,
DL, VT, S0->getOperand(0)),
19163 DAG.
getNode(ARMISD::VECTOR_REG_CAST,
DL, VT, S0->getOperand(1)),
19167 ARMISD::VMOVN,
DL, VT,
19168 DAG.
getNode(ARMISD::VECTOR_REG_CAST,
DL, VT, S0->getOperand(1)),
19169 DAG.
getNode(ARMISD::VECTOR_REG_CAST,
DL, VT, S0->getOperand(0)),
19177 return Op.getOpcode() == ISD::BUILD_VECTOR ||
19178 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
19179 (Op.getOpcode() == ISD::BITCAST &&
19180 Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR);
19183 for (
unsigned Op = 0;
Op <
N->getNumOperands();
Op++) {
19185 for (
unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
19203 int NumIns =
N->getNumOperands();
19204 assert((NumIns == 2 || NumIns == 4) &&
19205 "Expected 2 or 4 inputs to an MVETrunc");
19207 if (
N->getNumOperands() == 4)
19211 for (
int I = 0;
I < NumIns;
I++) {
19213 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
19218 Ptr, MPI, StoreVT,
Align(4));
19233 if (!LD || !LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed())
19236 EVT FromVT = LD->getMemoryVT();
19237 EVT ToVT =
N->getValueType(0);
19244 unsigned NumElements = 0;
19245 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
19247 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
19249 assert(NumElements != 0);
19255 LD->getExtensionType() != NewExtType)
19262 SDValue BasePtr = LD->getBasePtr();
19263 Align Alignment = LD->getBaseAlign();
19282 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
19283 Alignment, MMOFlags, AAInfo);
19299 EVT VT =
N->getValueType(0);
19301 assert(
N->getNumValues() == 2 &&
"Expected MVEEXT with 2 elements");
19302 assert((VT == MVT::v4i32 || VT == MVT::v8i16) &&
"Unexpected MVEEXT type");
19304 EVT ExtVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
19306 auto Extend = [&](
SDValue V) {
19315 if (
N->getOperand(0).getOpcode() == ARMISD::VDUP) {
19316 SDValue Ext = Extend(
N->getOperand(0));
19324 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
19325 unsigned Rev = VT == MVT::v4i32 ? ARMISD::VREV32 : ARMISD::VREV16;
19329 auto CheckInregMask = [&](
int Start,
int Offset) {
19331 if (Mask[Start + Idx] >= 0 && Mask[Start + Idx] != Idx * 2 +
Offset)
19337 if (CheckInregMask(0, 0))
19339 else if (CheckInregMask(0, 1))
19340 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
19341 else if (CheckInregMask(0, Mask.size()))
19343 else if (CheckInregMask(0, Mask.size() + 1))
19344 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
19349 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
19353 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
19355 if (V0.getNode() !=
N ||
V1.getNode() !=
N)
19360 if (
N->getOperand(0)->getOpcode() ==
ISD::LOAD)
19371 int NumOuts =
N->getNumValues();
19372 assert((NumOuts == 2 || NumOuts == 4) &&
19373 "Expected 2 or 4 outputs to an MVEEXT");
19374 EVT LoadVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
19376 if (
N->getNumOperands() == 4)
19382 StackPtr, MPI,
Align(4));
19385 for (
int I = 0;
I < NumOuts;
I++) {
19387 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
19388 DAG.
getConstant(
I * 16 / NumOuts,
DL, StackPtr.getValueType()));
19393 VT, Chain, Ptr, MPI, LoadVT,
Align(4));
19402 switch (
N->getOpcode()) {
19426 case ARMISD::VMOVrs:
19464 case ARMISD::BRCOND:
19468 case ARMISD::CSINC:
19469 case ARMISD::CSINV:
19470 case ARMISD::CSNEG:
19485 case ARMISD::PREDICATE_CAST:
19487 case ARMISD::VECTOR_REG_CAST:
19498 case ARMISD::VADDVs:
19499 case ARMISD::VADDVu:
19500 case ARMISD::VADDLVs:
19501 case ARMISD::VADDLVu:
19502 case ARMISD::VADDLVAs:
19503 case ARMISD::VADDLVAu:
19504 case ARMISD::VMLAVs:
19505 case ARMISD::VMLAVu:
19506 case ARMISD::VMLALVs:
19507 case ARMISD::VMLALVu:
19508 case ARMISD::VMLALVAs:
19509 case ARMISD::VMLALVAu:
19511 case ARMISD::VMOVN:
19513 case ARMISD::VQMOVNs:
19514 case ARMISD::VQMOVNu:
19516 case ARMISD::VQDMULH:
19522 case ARMISD::SMULWB: {
19523 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19529 case ARMISD::SMULWT: {
19530 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19536 case ARMISD::SMLALBB:
19537 case ARMISD::QADD16b:
19538 case ARMISD::QSUB16b:
19539 case ARMISD::UQADD16b:
19540 case ARMISD::UQSUB16b: {
19541 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19548 case ARMISD::SMLALBT: {
19549 unsigned LowWidth =
N->getOperand(0).getValueType().getSizeInBits();
19551 unsigned HighWidth =
N->getOperand(1).getValueType().getSizeInBits();
19558 case ARMISD::SMLALTB: {
19559 unsigned HighWidth =
N->getOperand(0).getValueType().getSizeInBits();
19561 unsigned LowWidth =
N->getOperand(1).getValueType().getSizeInBits();
19568 case ARMISD::SMLALTT: {
19569 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19576 case ARMISD::QADD8b:
19577 case ARMISD::QSUB8b:
19578 case ARMISD::UQADD8b:
19579 case ARMISD::UQSUB8b: {
19580 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19588 if (
N->getOperand(1) ==
N->getOperand(2))
19589 return N->getOperand(1);
19593 switch (
N->getConstantOperandVal(1)) {
19594 case Intrinsic::arm_neon_vld1:
19595 case Intrinsic::arm_neon_vld1x2:
19596 case Intrinsic::arm_neon_vld1x3:
19597 case Intrinsic::arm_neon_vld1x4:
19598 case Intrinsic::arm_neon_vld2:
19599 case Intrinsic::arm_neon_vld3:
19600 case Intrinsic::arm_neon_vld4:
19601 case Intrinsic::arm_neon_vld2lane:
19602 case Intrinsic::arm_neon_vld3lane:
19603 case Intrinsic::arm_neon_vld4lane:
19604 case Intrinsic::arm_neon_vld2dup:
19605 case Intrinsic::arm_neon_vld3dup:
19606 case Intrinsic::arm_neon_vld4dup:
19607 case Intrinsic::arm_neon_vst1:
19608 case Intrinsic::arm_neon_vst1x2:
19609 case Intrinsic::arm_neon_vst1x3:
19610 case Intrinsic::arm_neon_vst1x4:
19611 case Intrinsic::arm_neon_vst2:
19612 case Intrinsic::arm_neon_vst3:
19613 case Intrinsic::arm_neon_vst4:
19614 case Intrinsic::arm_neon_vst2lane:
19615 case Intrinsic::arm_neon_vst3lane:
19616 case Intrinsic::arm_neon_vst4lane:
19618 case Intrinsic::arm_mve_vld2q:
19619 case Intrinsic::arm_mve_vld4q:
19620 case Intrinsic::arm_mve_vst2q:
19621 case Intrinsic::arm_mve_vst4q:
19638 unsigned *
Fast)
const {
19644 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
19647 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19649 if (AllowsUnaligned) {
19651 *
Fast = Subtarget->hasV7Ops();
19656 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19660 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) {
19667 if (!Subtarget->hasMVEIntegerOps())
19671 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19672 Ty == MVT::v2i1)) {
19680 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19696 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19697 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19698 Ty == MVT::v2f64) {
19711 if ((
Op.isMemcpyOrMemmove() ||
Op.isZeroMemset()) && Subtarget->hasNEON() &&
19712 !FuncAttributes.hasFnAttr(Attribute::NoImplicitFloat)) {
19714 if (
Op.size() >= 16 &&
19720 }
else if (
Op.size() >= 8 &&
19737 if (!SrcTy->isIntegerTy() || !DstTy->
isIntegerTy())
19739 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
19741 return (SrcBits == 64 && DestBits == 32);
19750 return (SrcBits == 64 && DestBits == 32);
19786 return Subtarget->hasFullFP16();
19793 if (!Subtarget->hasMVEIntegerOps())
19812 if (Ld->isExpandingLoad())
19816 if (Subtarget->hasMVEIntegerOps())
19829 U->getOpcode() ==
ISD::SHL || U->getOpcode() == ARMISD::VSHLIMM))
19861bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(
const MachineFunction &MF,
19890 unsigned Scale = 1;
19907 if ((
V & (Scale - 1)) != 0)
19922 bool IsNeg =
false;
19928 unsigned NumBytes = std::max((
unsigned)VT.
getSizeInBits() / 8, 1U);
19953 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19983 default:
return false;
20002 int Scale = AM.
Scale;
20007 default:
return false;
20015 Scale = Scale & ~1;
20016 return Scale == 2 || Scale == 4 || Scale == 8;
20033 if (Scale & 1)
return false;
20040 const int Scale = AM.
Scale;
20050 return (Scale == 1) || (!AM.
HasBaseReg && Scale == 2);
20066 switch (AM.
Scale) {
20077 if (Subtarget->isThumb1Only())
20080 if (Subtarget->isThumb2())
20083 int Scale = AM.
Scale;
20085 default:
return false;
20089 if (Scale < 0) Scale = -Scale;
20097 if (Scale == 1 || (AM.
HasBaseReg && Scale == -1))
20110 if (Scale & 1)
return false;
20123 if (!Subtarget->isThumb())
20126 if (Subtarget->isThumb2())
20130 return Imm >= 0 &&
Imm <= 255;
20140 if (!Subtarget->isThumb())
20142 if (Subtarget->isThumb2())
20145 return AbsImm <= 255;
20180 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
20184 int RHSC = (int)
RHS->getZExtValue();
20185 if (RHSC < 0 && RHSC > -256) {
20195 }
else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
20198 int RHSC = (int)
RHS->getZExtValue();
20199 if (RHSC < 0 && RHSC > -0x1000) {
20241 int RHSC = (int)
RHS->getZExtValue();
20242 if (RHSC < 0 && RHSC > -0x100) {
20247 }
else if (RHSC > 0 && RHSC < 0x100) {
20258 bool isSEXTLoad,
bool IsMasked,
bool isLE,
20269 bool CanChangeType = isLE && !IsMasked;
20272 int RHSC = (int)
RHS->getZExtValue();
20274 auto IsInRange = [&](
int RHSC,
int Limit,
int Scale) {
20275 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
20280 }
else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
20291 if (VT == MVT::v4i16) {
20292 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
20294 }
else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
20295 if (IsInRange(RHSC, 0x80, 1))
20297 }
else if (Alignment >= 4 &&
20298 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
20299 IsInRange(RHSC, 0x80, 4))
20301 else if (Alignment >= 2 &&
20302 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
20303 IsInRange(RHSC, 0x80, 2))
20305 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
20318 if (Subtarget->isThumb1Only())
20325 bool isSEXTLoad =
false;
20326 bool IsMasked =
false;
20328 Ptr = LD->getBasePtr();
20329 VT = LD->getMemoryVT();
20330 Alignment = LD->getAlign();
20331 AS = LD->getAddressSpace();
20334 Ptr = ST->getBasePtr();
20335 VT = ST->getMemoryVT();
20336 Alignment = ST->getAlign();
20337 AS = ST->getAddressSpace();
20339 Ptr = LD->getBasePtr();
20340 VT = LD->getMemoryVT();
20341 Alignment = LD->getAlign();
20342 AS = LD->getAddressSpace();
20346 Ptr = ST->getBasePtr();
20347 VT = ST->getMemoryVT();
20348 Alignment = ST->getAlign();
20349 AS = ST->getAddressSpace();
20364 bool isLegal =
false;
20366 isLegal = Subtarget->hasMVEIntegerOps() &&
20368 Ptr.
getNode(), VT, Alignment, isSEXTLoad, IsMasked,
20369 Subtarget->isLittle(),
Base,
Offset, isInc, DAG);
20371 if (Subtarget->isThumb2())
20396 bool isSEXTLoad =
false, isNonExt;
20397 bool IsMasked =
false;
20399 VT = LD->getMemoryVT();
20400 Ptr = LD->getBasePtr();
20401 Alignment = LD->getAlign();
20405 VT = ST->getMemoryVT();
20406 Ptr = ST->getBasePtr();
20407 Alignment = ST->getAlign();
20408 isNonExt = !ST->isTruncatingStore();
20410 VT = LD->getMemoryVT();
20411 Ptr = LD->getBasePtr();
20412 Alignment = LD->getAlign();
20417 VT = ST->getMemoryVT();
20418 Ptr = ST->getBasePtr();
20419 Alignment = ST->getAlign();
20420 isNonExt = !ST->isTruncatingStore();
20425 if (Subtarget->isThumb1Only()) {
20428 assert(
Op->getValueType(0) == MVT::i32 &&
"Non-i32 post-inc op?!");
20429 if (
Op->getOpcode() !=
ISD::ADD || !isNonExt)
20432 if (!RHS || RHS->getZExtValue() != 4)
20434 if (Alignment <
Align(4))
20438 Base =
Op->getOperand(0);
20444 bool isLegal =
false;
20446 isLegal = Subtarget->hasMVEIntegerOps() &&
20451 if (Subtarget->isThumb2())
20465 !Subtarget->isThumb2())
20479 const APInt &DemandedElts,
20481 unsigned Depth)
const {
20484 switch (
Op.getOpcode()) {
20491 if (
Op.getResNo() == 0) {
20502 case ARMISD::CMOV: {
20505 if (
Known.isUnknown())
20517 case Intrinsic::arm_ldaex:
20518 case Intrinsic::arm_ldrex: {
20526 case ARMISD::BFI: {
20533 const APInt &Mask =
Op.getConstantOperandAPInt(2);
20534 Known.Zero &= Mask;
20538 case ARMISD::VGETLANEs:
20539 case ARMISD::VGETLANEu: {
20540 const SDValue &SrcSV =
Op.getOperand(0);
20546 "VGETLANE index out of bounds");
20551 EVT VT =
Op.getValueType();
20557 if (
Op.getOpcode() == ARMISD::VGETLANEs)
20565 case ARMISD::VMOVrh: {
20571 case ARMISD::CSINC:
20572 case ARMISD::CSINV:
20573 case ARMISD::CSNEG: {
20581 if (
Op.getOpcode() == ARMISD::CSINC)
20584 else if (
Op.getOpcode() == ARMISD::CSINV)
20586 else if (
Op.getOpcode() == ARMISD::CSNEG)
20593 case ARMISD::VORRIMM:
20594 case ARMISD::VBICIMM: {
20595 unsigned Encoded =
Op.getConstantOperandVal(1);
20596 unsigned DecEltBits = 0;
20599 unsigned EltBits =
Op.getScalarValueSizeInBits();
20600 if (EltBits != DecEltBits) {
20609 bool IsVORR =
Op.getOpcode() == ARMISD::VORRIMM;
20610 APInt Imm(DecEltBits, DecodedVal);
20638 if (
Imm == 0 ||
Imm == ~0U)
20641 unsigned Opc =
Op.getOpcode();
20643 EVT VT =
Op.getValueType();
20645 unsigned ShrunkImm =
Imm & Demanded;
20646 unsigned ExpandedImm =
Imm | ~Demanded;
20648 auto IsLegalImm = [ShrunkImm, ExpandedImm](
unsigned CandidateImm) ->
bool {
20649 return (ShrunkImm & CandidateImm) == ShrunkImm &&
20650 (~ExpandedImm & CandidateImm) == 0;
20652 auto UseImm = [
Imm,
Opc,
Op, VT, &TLO](
unsigned NewImm) ->
bool {
20664 if (ShrunkImm == 0) {
20665 ++NumOptimizedImms;
20666 return UseImm(ShrunkImm);
20672 if (ExpandedImm == ~0U) {
20673 ++NumOptimizedImms;
20674 return UseImm(ExpandedImm);
20682 if (IsLegalImm(0xFF)) {
20683 ++NumOptimizedImms;
20684 return UseImm(0xFF);
20687 if (IsLegalImm(0xFFFF)) {
20688 ++NumOptimizedImms;
20689 return UseImm(0xFFFF);
20703 ++NumOptimizedImms;
20704 return UseImm(ShrunkImm);
20712 if ((~ExpandedImm) < 256) {
20713 ++NumOptimizedImms;
20714 return UseImm(ExpandedImm);
20721 ++NumOptimizedImms;
20722 return UseImm(ExpandedImm);
20741 EVT VT =
Op.getValueType();
20756 switch (
Op.getOpcode()) {
20767 unsigned Imm =
C->getZExtValue();
20774 unsigned Depth)
const {
20775 unsigned Opc =
Op.getOpcode();
20779 case ARMISD::LSRL: {
20783 if (
Op.getResNo() == 0 && !
Op->hasAnyUseOfValue(1) &&
20785 unsigned ShAmt =
Op->getConstantOperandVal(2);
20795 case ARMISD::VBICIMM: {
20797 unsigned ModImm =
Op.getConstantOperandVal(1);
20798 unsigned EltBits = 0;
20800 if ((OriginalDemandedBits & Mask) == 0)
20806 Op, OriginalDemandedBits, OriginalDemandedElts,
Known, TLO,
Depth);
20821 if (!Subtarget->hasVFP2Base())
20825 if (ConstraintVT.
isVector() && Subtarget->hasNEON() &&
20837 unsigned S = Constraint.
size();
20839 switch (Constraint[0]) {
20851 }
else if (S == 2) {
20852 switch (Constraint[0]) {
20869 Value *CallOperandVal =
info.CallOperandVal;
20872 if (!CallOperandVal)
20876 switch (*constraint) {
20882 if (Subtarget->isThumb())
20897 if (PR == 0 || VT == MVT::Other)
20899 if (ARM::SPRRegClass.
contains(PR))
20900 return VT != MVT::f32 && VT != MVT::f16 && VT != MVT::i32;
20901 if (ARM::DPRRegClass.
contains(PR))
20906using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20910 switch (Constraint.
size()) {
20913 switch (Constraint[0]) {
20915 if (Subtarget->isThumb())
20916 return RCPair(0U, &ARM::tGPRRegClass);
20917 return RCPair(0U, &ARM::GPRRegClass);
20919 if (Subtarget->isThumb())
20920 return RCPair(0U, &ARM::hGPRRegClass);
20923 if (Subtarget->isThumb1Only())
20924 return RCPair(0U, &ARM::tGPRRegClass);
20925 return RCPair(0U, &ARM::GPRRegClass);
20927 if (VT == MVT::Other)
20929 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20930 return RCPair(0U, &ARM::SPRRegClass);
20932 return RCPair(0U, &ARM::DPRRegClass);
20934 return RCPair(0U, &ARM::QPRRegClass);
20937 if (VT == MVT::Other)
20939 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20940 return RCPair(0U, &ARM::SPR_8RegClass);
20942 return RCPair(0U, &ARM::DPR_8RegClass);
20944 return RCPair(0U, &ARM::QPR_8RegClass);
20947 if (VT == MVT::Other)
20949 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20950 return RCPair(0U, &ARM::SPRRegClass);
20952 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20954 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20960 if (Constraint[0] ==
'T') {
20961 switch (Constraint[1]) {
20965 return RCPair(0U, &ARM::tGPREvenRegClass);
20967 return RCPair(0U, &ARM::tGPROddRegClass);
20976 if (
StringRef(
"{cc}").equals_insensitive(Constraint))
20977 return std::make_pair(
unsigned(ARM::CPSR), &ARM::CCRRegClass);
20980 if (
StringRef(
"{r14}").equals_insensitive(Constraint))
20981 Constraint =
"{lr}";
20985 return {0,
nullptr};
20993 std::vector<SDValue> &
Ops,
20998 if (Constraint.
size() != 1)
21001 char ConstraintLetter = Constraint[0];
21002 switch (ConstraintLetter) {
21005 case 'I':
case 'J':
case 'K':
case 'L':
21006 case 'M':
case 'N':
case 'O':
21011 int64_t CVal64 =
C->getSExtValue();
21012 int CVal = (int) CVal64;
21015 if (CVal != CVal64)
21018 switch (ConstraintLetter) {
21022 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
21023 if (CVal >= 0 && CVal <= 65535)
21027 if (Subtarget->isThumb1Only()) {
21030 if (CVal >= 0 && CVal <= 255)
21032 }
else if (Subtarget->isThumb2()) {
21046 if (Subtarget->isThumb1Only()) {
21051 if (CVal >= -255 && CVal <= -1)
21057 if (CVal >= -4095 && CVal <= 4095)
21063 if (Subtarget->isThumb1Only()) {
21070 }
else if (Subtarget->isThumb2()) {
21090 if (Subtarget->isThumb1Only()) {
21093 if (CVal >= -7 && CVal < 7)
21095 }
else if (Subtarget->isThumb2()) {
21115 if (Subtarget->isThumb1Only()) {
21118 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
21124 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
21130 if (Subtarget->isThumb1Only()) {
21132 if (CVal >= 0 && CVal <= 31)
21138 if (Subtarget->isThumb1Only()) {
21141 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
21150 if (Result.getNode()) {
21151 Ops.push_back(Result);
21161 "Unhandled Opcode in getDivRemLibcall");
21167 case MVT::i8: LC =
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
21168 case MVT::i16: LC =
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
21169 case MVT::i32: LC =
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
21170 case MVT::i64: LC =
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
21177 const AttributeList &FuncAttrs, RTLIB::LibcallImpl LCImpl) {
21180 "Unhandled Opcode in getDivRemArgList");
21181 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1)};
21185 case RTLIB::impl___rt_sdiv:
21186 case RTLIB::impl___rt_udiv:
21187 case RTLIB::impl___rt_sdiv64:
21188 case RTLIB::impl___rt_udiv64:
21199 assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||
21200 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() ||
21201 Subtarget->isTargetFuchsia() || Subtarget->isTargetWindows()) &&
21202 "Register-based DivRem lowering only");
21203 unsigned Opcode =
Op->getOpcode();
21205 "Invalid opcode for Div/Rem lowering");
21207 EVT VT =
Op->getValueType(0);
21227 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
21228 : Subtarget->hasDivideInARMMode();
21229 if (hasDivide &&
Op->getValueType(0).isSimple() &&
21230 Op->getSimpleValueType(0) == MVT::i32) {
21232 const SDValue Dividend =
Op->getOperand(0);
21233 const SDValue Divisor =
Op->getOperand(1);
21234 SDValue Div = DAG.
getNode(DivOpcode, dl, VT, Dividend, Divisor);
21245 if (LCImpl == RTLIB::Unsupported)
21248 auto [FuncTy, FuncAttrs] =
21252 Type *RetTy = FuncTy->getReturnType();
21262 if (
getTM().getTargetTriple().isOSWindows())
21265 TargetLowering::CallLoweringInfo CLI(DAG);
21269 Callee, std::move(Args))
21274 std::pair<SDValue, SDValue> CallInfo =
LowerCallTo(CLI);
21275 return CallInfo.first;
21281 EVT VT =
N->getValueType(0);
21287 Result[0], Result[1]);
21293 if (LCImpl == RTLIB::Unsupported)
21296 auto [FuncTy, FuncAttrs] =
21300 Type *RetTy = FuncTy->getReturnType();
21310 if (
getTM().getTargetTriple().isOSWindows())
21317 Callee, std::move(Args))
21321 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
21324 SDNode *ResNode = CallResult.first.getNode();
21331 assert(
getTM().getTargetTriple().isOSWindows() &&
21332 "unsupported target platform");
21340 "no-stack-arg-probe")) {
21344 Chain =
SP.getValue(1);
21361 SDVTList NodeTys = DAG.
getVTList(MVT::Other, MVT::Glue);
21362 Chain = DAG.
getNode(ARMISD::WIN__CHKSTK,
DL, NodeTys, Chain, Glue);
21372 bool IsStrict =
Op->isStrictFPOpcode();
21373 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
21374 EVT DstTy =
Op.getValueType();
21378 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
21379 "Unexpected type for custom-lowering FP_EXTEND");
21382 if (DstTy == MVT::f32 && SrcTy == MVT::f16 && Subtarget->hasFP16())
21385 if (DstTy == MVT::f64 && SrcTy == MVT::f16 && Subtarget->hasFP16() &&
21386 Subtarget->hasFP64() && Subtarget->hasFPARMv8Base())
21390 if (SrcTy == MVT::bf16) {
21396 if (DstTy == MVT::f32)
21399 assert(DstTy == MVT::f64 &&
"Unexpected BF16 convert");
21402 {
Op.getOperand(0), Res});
21407 if (SrcTy == MVT::f32 && DstTy == MVT::f64 && Subtarget->hasFP64()) {
21426 for (
unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
21427 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
21428 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
21429 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
21433 {DstVT, MVT::Other}, {Chain, SrcVal});
21440 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
21441 "Unexpected type for custom-lowering FP_EXTEND");
21442 std::tie(SrcVal, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
21447 return IsStrict ? DAG.
getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
21451 bool IsStrict =
Op->isStrictFPOpcode();
21453 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
21455 EVT DstVT =
Op.getValueType();
21457 if (DstVT == MVT::bf16) {
21458 if (Subtarget->hasBF16() && SrcVT == MVT::f32)
21463 const unsigned DstSz =
Op.getValueType().getSizeInBits();
21466 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
21467 "Unexpected type for custom-lowering FP_ROUND");
21469 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) &&
21470 "With both FP DP and 16, any FP conversion is legal!");
21475 if (SrcSz == 32 && Subtarget->hasFP16())
21480 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
21481 "Unexpected type for custom-lowering FP_ROUND");
21485 std::tie(Result, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
21497 if (v == 0xffffffff)
21508 bool ForCodeSize)
const {
21509 if (!Subtarget->hasVFP3Base())
21511 if (VT == MVT::f16 && Subtarget->hasFullFP16())
21514 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
21517 if (VT == MVT::f32)
21520 if (VT == MVT::f64 && Subtarget->hasFP64())
21528 bool ForCodeSize)
const {
21529 if (Subtarget->useSoftFloat() || !Subtarget->hasVFP2Base())
21542 case Intrinsic::arm_neon_vld1:
21543 case Intrinsic::arm_neon_vld2:
21544 case Intrinsic::arm_neon_vld3:
21545 case Intrinsic::arm_neon_vld4:
21546 case Intrinsic::arm_neon_vld2lane:
21547 case Intrinsic::arm_neon_vld3lane:
21548 case Intrinsic::arm_neon_vld4lane:
21549 case Intrinsic::arm_neon_vld2dup:
21550 case Intrinsic::arm_neon_vld3dup:
21551 case Intrinsic::arm_neon_vld4dup: {
21554 auto &
DL =
I.getDataLayout();
21555 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21557 Info.ptrVal =
I.getArgOperand(0);
21559 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21566 case Intrinsic::arm_neon_vld1x2:
21567 case Intrinsic::arm_neon_vld1x3:
21568 case Intrinsic::arm_neon_vld1x4: {
21571 auto &
DL =
I.getDataLayout();
21572 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21574 Info.ptrVal =
I.getArgOperand(
I.arg_size() - 1);
21576 Info.align =
I.getParamAlign(
I.arg_size() - 1).valueOrOne();
21582 case Intrinsic::arm_neon_vst1:
21583 case Intrinsic::arm_neon_vst2:
21584 case Intrinsic::arm_neon_vst3:
21585 case Intrinsic::arm_neon_vst4:
21586 case Intrinsic::arm_neon_vst2lane:
21587 case Intrinsic::arm_neon_vst3lane:
21588 case Intrinsic::arm_neon_vst4lane: {
21591 auto &
DL =
I.getDataLayout();
21592 unsigned NumElts = 0;
21593 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21594 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21597 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21600 Info.ptrVal =
I.getArgOperand(0);
21602 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21609 case Intrinsic::arm_neon_vst1x2:
21610 case Intrinsic::arm_neon_vst1x3:
21611 case Intrinsic::arm_neon_vst1x4: {
21614 auto &
DL =
I.getDataLayout();
21615 unsigned NumElts = 0;
21616 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21617 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21620 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21623 Info.ptrVal =
I.getArgOperand(0);
21625 Info.align =
I.getParamAlign(0).valueOrOne();
21631 case Intrinsic::arm_mve_vld2q:
21632 case Intrinsic::arm_mve_vld4q: {
21636 unsigned Factor =
Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21638 Info.ptrVal =
I.getArgOperand(0);
21646 case Intrinsic::arm_mve_vst2q:
21647 case Intrinsic::arm_mve_vst4q: {
21650 Type *VecTy =
I.getArgOperand(1)->getType();
21651 unsigned Factor =
Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21653 Info.ptrVal =
I.getArgOperand(0);
21661 case Intrinsic::arm_mve_vldr_gather_base:
21662 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21664 Info.ptrVal =
nullptr;
21666 Info.align =
Align(1);
21671 case Intrinsic::arm_mve_vldr_gather_base_wb:
21672 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21674 Info.ptrVal =
nullptr;
21675 Info.memVT =
MVT::getVT(
I.getType()->getContainedType(0));
21676 Info.align =
Align(1);
21681 case Intrinsic::arm_mve_vldr_gather_offset:
21682 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21684 Info.ptrVal =
nullptr;
21689 Info.align =
Align(1);
21694 case Intrinsic::arm_mve_vstr_scatter_base:
21695 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21697 Info.ptrVal =
nullptr;
21698 Info.memVT =
MVT::getVT(
I.getArgOperand(2)->getType());
21699 Info.align =
Align(1);
21704 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21705 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21707 Info.ptrVal =
nullptr;
21708 Info.memVT =
MVT::getVT(
I.getArgOperand(2)->getType());
21709 Info.align =
Align(1);
21714 case Intrinsic::arm_mve_vstr_scatter_offset:
21715 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21717 Info.ptrVal =
nullptr;
21722 Info.align =
Align(1);
21727 case Intrinsic::arm_ldaex:
21728 case Intrinsic::arm_ldrex: {
21729 auto &
DL =
I.getDataLayout();
21730 Type *ValTy =
I.getParamElementType(0);
21733 Info.ptrVal =
I.getArgOperand(0);
21735 Info.align =
DL.getABITypeAlign(ValTy);
21740 case Intrinsic::arm_stlex:
21741 case Intrinsic::arm_strex: {
21742 auto &
DL =
I.getDataLayout();
21743 Type *ValTy =
I.getParamElementType(1);
21746 Info.ptrVal =
I.getArgOperand(1);
21748 Info.align =
DL.getABITypeAlign(ValTy);
21753 case Intrinsic::arm_stlexd:
21754 case Intrinsic::arm_strexd:
21756 Info.memVT = MVT::i64;
21757 Info.ptrVal =
I.getArgOperand(2);
21759 Info.align =
Align(8);
21764 case Intrinsic::arm_ldaexd:
21765 case Intrinsic::arm_ldrexd:
21767 Info.memVT = MVT::i64;
21768 Info.ptrVal =
I.getArgOperand(0);
21770 Info.align =
Align(8);
21784 assert(Ty->isIntegerTy());
21786 unsigned Bits = Ty->getPrimitiveSizeInBits();
21787 if (Bits == 0 || Bits > 32)
21794 unsigned Index)
const {
21806 if (!Subtarget->hasDataBarrier()) {
21810 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21811 Value*
args[6] = {Builder.getInt32(15), Builder.getInt32(0),
21812 Builder.getInt32(0), Builder.getInt32(7),
21813 Builder.getInt32(10), Builder.getInt32(5)};
21814 return Builder.CreateIntrinsicWithoutFolding(Intrinsic::arm_mcr,
args);
21823 return Builder.CreateIntrinsicWithoutFolding(Intrinsic::arm_dmb, CDomain);
21844 if (Subtarget->preferISHSTBarriers())
21878 if (Subtarget->isMClass())
21879 has64BitLLSC =
false;
21880 else if (Subtarget->isThumb())
21881 has64BitLLSC = Subtarget->hasV7Ops();
21883 has64BitLLSC = Subtarget->hasV6Ops();
21885 unsigned Size =
SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21886 if (
Size == 64 && has64BitLLSC)
21906 if (Subtarget->isMClass())
21907 has64BitLLSC =
false;
21908 else if (Subtarget->isThumb())
21909 has64BitLLSC = Subtarget->hasV7Ops();
21911 has64BitLLSC = Subtarget->hasV6Ops();
21914 if (
Size == 64 && has64BitLLSC)
21933 if (Subtarget->isMClass())
21934 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21935 else if (Subtarget->isThumb())
21936 hasAtomicRMW = Subtarget->hasV7Ops();
21938 hasAtomicRMW = Subtarget->hasV6Ops();
21939 if (
Size <= (Subtarget->isMClass() ? 32U : 64U) && hasAtomicRMW) {
21963 bool HasAtomicCmpXchg;
21964 if (Subtarget->isMClass())
21965 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21966 else if (Subtarget->isThumb())
21967 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21969 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21971 HasAtomicCmpXchg &&
Size <= (Subtarget->isMClass() ? 32U : 64U))
21978 return InsertFencesForAtomic;
21982 if (Subtarget->isTargetFuchsia())
21985 return !Subtarget->isROPI() && !Subtarget->isRWPI();
21993 if (Subtarget->isTargetFuchsia()) {
21996 M, Intrinsic::thread_pointer, IRB.
getPtrTy());
22007 RTLIB::LibcallImpl SecurityCheckCookieLibcall =
22008 Libcalls.getLibcallImpl(RTLIB::SECURITY_CHECK_COOKIE);
22010 RTLIB::LibcallImpl SecurityCookieVar =
22011 Libcalls.getLibcallImpl(RTLIB::STACK_CHECK_GUARD);
22012 if (SecurityCheckCookieLibcall != RTLIB::Unsupported &&
22013 SecurityCookieVar != RTLIB::Unsupported) {
22022 F->addParamAttr(0, Attribute::AttrKind::InReg);
22029 unsigned &
Cost)
const {
22031 if (!Subtarget->hasNEON())
22060 unsigned Opcode =
Op.getOpcode();
22062 case ARMISD::VORRIMM:
22063 case ARMISD::VBICIMM:
22067 Op, DemandedElts, DAG, Kind, ConsiderFlags,
Depth);
22071 return Subtarget->hasV5TOps() && !Subtarget->isThumb1Only();
22075 return Subtarget->hasV5TOps() && !Subtarget->isThumb1Only();
22080 if (!Subtarget->hasV7Ops())
22086 if (!Mask || Mask->getValue().getBitWidth() > 32u)
22088 auto MaskVal =
unsigned(Mask->getValue().getZExtValue());
22096 if (Subtarget->hasMinSize() && !
getTM().getTargetTriple().isOSWindows())
22105 Module *M = Builder.getModule();
22111 if (ValueTy->getPrimitiveSizeInBits() == 64) {
22113 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
22116 Builder.CreateIntrinsic(
Int, Addr,
nullptr,
"lohi");
22118 Value *
Lo = Builder.CreateExtractValue(LoHi, 0,
"lo");
22119 Value *
Hi = Builder.CreateExtractValue(LoHi, 1,
"hi");
22120 if (!Subtarget->isLittle())
22122 Lo = Builder.CreateZExt(
Lo, ValueTy,
"lo64");
22123 Hi = Builder.CreateZExt(
Hi, ValueTy,
"hi64");
22124 return Builder.CreateOr(
22125 Lo, Builder.CreateShl(
Hi, ConstantInt::get(ValueTy, 32)),
"val64");
22129 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
22130 CallInst *CI = Builder.CreateIntrinsicWithoutFolding(
Int, Tys, Addr);
22133 0,
Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
22134 return Builder.CreateTruncOrBitCast(CI, ValueTy);
22139 if (!(Subtarget->isThumb() ? Subtarget->hasV7Clrex()
22140 : Subtarget->hasV6KOps()))
22142 Builder.CreateIntrinsic(Intrinsic::arm_clrex, {});
22148 Module *M = Builder.getModule();
22156 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
22159 Value *
Lo = Builder.CreateTrunc(Val, Int32Ty,
"lo");
22160 Value *
Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 32), Int32Ty,
"hi");
22161 if (!Subtarget->isLittle())
22163 return Builder.CreateIntrinsic(
Int, {
Lo,
Hi, Addr});
22166 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
22170 CallInst *CI = Builder.CreateCall(
22171 Strex, {Builder.CreateZExtOrBitCast(
22181 return Subtarget->isMClass();
22189 return (
DL.getTypeSizeInBits(VecTy) + 127) / 128;
22196 unsigned VecSize =
DL.getTypeSizeInBits(VecTy);
22199 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
22207 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
22215 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
22218 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
22223 if (Subtarget->hasNEON() && VecSize == 64)
22225 return VecSize % 128 == 0;
22229 if (Subtarget->hasNEON())
22231 if (Subtarget->hasMVEIntegerOps())
22251 "Invalid interleave factor");
22252 assert(!Shuffles.
empty() &&
"Empty shufflevector input");
22254 "Unmatched number of shufflevectors and indices");
22259 assert(!Mask && GapMask.
popcount() == Factor &&
"Unexpected mask on a load");
22262 Type *EltTy = VecTy->getElementType();
22265 Align Alignment = LI->getAlign();
22281 Builder.disableGEPCanonicalization();
22284 Value *BaseAddr = LI->getPointerOperand();
22286 if (NumLoads > 1) {
22290 VecTy->getNumElements() / NumLoads);
22296 if (Subtarget->hasNEON()) {
22297 Type *PtrTy = Builder.getPtrTy(LI->getPointerAddressSpace());
22298 Type *Tys[] = {VecTy, PtrTy};
22299 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
22300 Intrinsic::arm_neon_vld3,
22301 Intrinsic::arm_neon_vld4};
22304 Ops.push_back(BaseAddr);
22305 Ops.push_back(Builder.getInt32(LI->getAlign().value()));
22307 return Builder.CreateIntrinsic(LoadInts[Factor - 2], Tys,
Ops,
22310 assert((Factor == 2 || Factor == 4) &&
22311 "expected interleave factor of 2 or 4 for MVE");
22313 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
22314 Type *PtrTy = Builder.getPtrTy(LI->getPointerAddressSpace());
22315 Type *Tys[] = {VecTy, PtrTy};
22318 Ops.push_back(BaseAddr);
22319 return Builder.CreateIntrinsic(LoadInts, Tys,
Ops,
nullptr,
22329 for (
unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
22333 BaseAddr = Builder.CreateConstGEP1_32(VecTy->getElementType(), BaseAddr,
22334 VecTy->getNumElements() * Factor);
22340 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
22342 unsigned Index = Indices[i];
22344 Value *SubVec = Builder.CreateExtractValue(VldN, Index);
22348 SubVec = Builder.CreateIntToPtr(
22352 SubVecs[SV].push_back(SubVec);
22361 auto &SubVec = SubVecs[SVI];
22364 SVI->replaceAllUsesWith(WideVec);
22400 const APInt &GapMask)
const {
22402 "Invalid interleave factor");
22407 "Unexpected mask on store");
22410 assert(VecTy->getNumElements() % Factor == 0 &&
"Invalid interleaved store");
22412 unsigned LaneLen = VecTy->getNumElements() / Factor;
22413 Type *EltTy = VecTy->getElementType();
22417 Align Alignment =
SI->getAlign();
22430 Builder.disableGEPCanonicalization();
22435 Type *IntTy =
DL.getIntPtrType(EltTy);
22440 Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
22441 Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
22447 Value *BaseAddr =
SI->getPointerOperand();
22449 if (NumStores > 1) {
22452 LaneLen /= NumStores;
22462 if (Subtarget->hasNEON()) {
22463 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
22464 Intrinsic::arm_neon_vst3,
22465 Intrinsic::arm_neon_vst4};
22466 Type *PtrTy = Builder.getPtrTy(
SI->getPointerAddressSpace());
22470 Ops.push_back(BaseAddr);
22472 Ops.push_back(Builder.getInt32(
SI->getAlign().value()));
22473 Builder.CreateIntrinsic(StoreInts[Factor - 2], Tys,
Ops);
22475 assert((Factor == 2 || Factor == 4) &&
22476 "expected interleave factor of 2 or 4 for MVE");
22478 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
22479 Type *PtrTy = Builder.getPtrTy(
SI->getPointerAddressSpace());
22483 Ops.push_back(BaseAddr);
22485 for (
unsigned F = 0;
F < Factor;
F++) {
22486 Ops.push_back(Builder.getInt32(
F));
22487 Builder.CreateIntrinsic(StoreInts, Tys,
Ops);
22493 for (
unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
22496 if (StoreCount > 0)
22497 BaseAddr = Builder.CreateConstGEP1_32(
SubVecTy->getElementType(),
22498 BaseAddr, LaneLen * Factor);
22503 for (
unsigned i = 0; i < Factor; i++) {
22504 unsigned IdxI = StoreCount * LaneLen * Factor + i;
22505 if (Mask[IdxI] >= 0) {
22506 Shuffles.
push_back(Builder.CreateShuffleVector(
22509 unsigned StartMask = 0;
22510 for (
unsigned j = 1; j < LaneLen; j++) {
22511 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
22512 if (Mask[IdxJ * Factor + IdxI] >= 0) {
22513 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
22523 Shuffles.
push_back(Builder.CreateShuffleVector(
22544 for (
unsigned i = 0; i < ST->getNumElements(); ++i) {
22548 Members += SubMembers;
22554 Members += SubMembers * AT->getNumElements();
22555 }
else if (Ty->isFloatTy()) {
22560 }
else if (Ty->isDoubleTy()) {
22572 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
22574 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
22576 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
22589 return (Members > 0 && Members <= 4);
22595 const Align ABITypeAlign =
DL.getABITypeAlign(ArgTy);
22597 return ABITypeAlign;
22602 assert(StackAlign &&
"data layout string is missing stack alignment");
22603 return std::min(ABITypeAlign, *StackAlign);
22617 uint64_t Members = 0;
22621 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy();
22622 return IsHA || IsIntArray;
22645void ARMTargetLowering::insertCopiesSplitCSR(
22649 const MCPhysReg *IStart =
TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
22659 RC = &ARM::GPRRegClass;
22660 else if (ARM::DPRRegClass.
contains(*
I))
22661 RC = &ARM::DPRRegClass;
22671 assert(Entry->getParent()->getFunction().hasFnAttribute(
22672 Attribute::NoUnwind) &&
22673 "Function should be nounwind in insertCopiesSplitCSR!");
22674 Entry->addLiveIn(*
I);
22679 for (
auto *Exit : Exits)
22681 TII->get(TargetOpcode::COPY), *
I)
22692 return Subtarget->hasMVEIntegerOps();
22702 unsigned NumElements = VTy->getNumElements();
22709 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22710 return Subtarget->hasMVEFloatOps();
22715 return Subtarget->hasMVEIntegerOps() &&
22716 (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
22717 ScalarTy->isIntegerTy(32));
22721 static const MCPhysReg RCRegs[] = {ARM::FPSCR_RM};
22732 unsigned TyWidth = Ty->getScalarSizeInBits() * Ty->getNumElements();
22734 assert(TyWidth >= 128 &&
"Width of vector type must be at least 128 bits");
22736 if (TyWidth > 128) {
22737 int Stride = Ty->getNumElements() / 2;
22741 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22743 auto *LowerSplitA =
B.CreateShuffleVector(InputA, LowerSplitMask);
22744 auto *LowerSplitB =
B.CreateShuffleVector(InputB, LowerSplitMask);
22745 auto *UpperSplitA =
B.CreateShuffleVector(InputA, UpperSplitMask);
22746 auto *UpperSplitB =
B.CreateShuffleVector(InputB, UpperSplitMask);
22747 Value *LowerSplitAcc =
nullptr;
22748 Value *UpperSplitAcc =
nullptr;
22751 LowerSplitAcc =
B.CreateShuffleVector(
Accumulator, LowerSplitMask);
22752 UpperSplitAcc =
B.CreateShuffleVector(
Accumulator, UpperSplitMask);
22756 B, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
22758 B, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);
22760 ArrayRef<int> JoinMask(&SplitSeqVec[0], Ty->getNumElements());
22761 return B.CreateShuffleVector(LowerSplitInt, UpperSplitInt, JoinMask);
22768 ConstRotation = ConstantInt::get(IntTy, (
int)Rotation);
22771 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmlaq, Ty,
22773 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmulq, Ty,
22774 {ConstRotation, InputB, InputA});
22779 auto *ConstHalving = ConstantInt::get(IntTy, 1);
22782 ConstRotation = ConstantInt::get(IntTy, 0);
22784 ConstRotation = ConstantInt::get(IntTy, 1);
22786 if (!ConstRotation)
22789 return B.CreateIntrinsic(Intrinsic::arm_mve_vcaddq, Ty,
22790 {ConstHalving, ConstRotation, InputA, InputB});
static bool isAddSubSExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG, bool isSigned)
static SDValue carryFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG, bool Invert)
static SDValue overflowFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG)
static bool isZeroExtended(SDValue N, SelectionDAG &DAG)
static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG)
static const MCPhysReg GPRArgRegs[]
static SDValue valueToCarryFlag(SDValue Value, SelectionDAG &DAG, bool Invert)
constexpr MVT FlagsVT
Value type used for NZCV flags.
static unsigned getCmpOperandFoldingProfit(SDValue Op, bool AllowExtend)
Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm, const APInt &Demanded, TargetLowering::TargetLoweringOpt &TLO, unsigned NewOpc)
static bool isSafeSignedCMN(SDValue Op, SelectionDAG &DAG)
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
static bool isSignExtended(SDValue N, SelectionDAG &DAG)
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
static bool isAddSubZExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG)
static bool isStore(int Opcode)
static bool isThumb(const MCSubtargetInfo &STI)
static SDValue PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isIncompatibleReg(const MCPhysReg &PR, MVT VT)
static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVECTOR_DEINTERLEAVECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total value size to 64 bits.
static cl::opt< unsigned > ConstpoolPromotionMaxSize("arm-promote-constant-max-size", cl::Hidden, cl::desc("Maximum size of constant to promote into a constant pool"), cl::init(64))
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isVTBLMask(ArrayRef< int > M, EVT VT)
static SDValue PerformSUBCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
static cl::opt< bool > EnableConstpoolPromotion("arm-promote-constant", cl::Hidden, cl::desc("Enable / disable promotion of unnamed_addr constants into " "constant pools"), cl::init(false))
static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG)
static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
static SDValue PerformExtractEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static const APInt * isPowerOf2Constant(SDValue V)
static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG)
static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static bool isValidMVECond(unsigned CC, bool IsFloat)
static SDValue PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC)
IntCCToARMCC - Convert a DAG integer condition code to an ARM CC.
static SDValue PerformSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSTORECombine - Target-specific dag combine xforms for ISD::STORE.
static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isGTorGE(ISD::CondCode CC)
static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a vldN-lane (N > 1) intrinsic,...
static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask)
static bool isReverseMask(ArrayRef< int > M, EVT VT)
static SDValue PerformLegalizedVECTOR_DEINTERLEAVECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVMulVCTPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVMulVCTPCombine - VCVT (fixed-point to floating-point, Advanced SIMD) can replace combinations...
static SDValue createGPRPairNode2xi32(SelectionDAG &DAG, SDValue V0, SDValue V1)
static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG)
static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc)
static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool CanInvertMVEVCMP(SDValue N)
static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG)
static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformShiftCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
PerformShiftCombine - Checks for immediate versions of vector shifts and lowers them.
static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, ARMCC::CondCodes &CondCode2)
FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static SDValue PerformVMOVrsCombine(SDNode *N, SelectionDAG &DAG)
static EVT getVectorTyFromPredicateVector(EVT VT)
static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG)
static SDValue handleCMSEValue(const SDValue &Value, const ISD::InputArg &Arg, SelectionDAG &DAG, const SDLoc &DL)
static SDValue PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
static bool isSRL16(const SDValue &Op)
static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC)
static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr, SDValue Inc, const SelectionDAG &DAG)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static Register genTPEntry(MachineBasicBlock *TpEntry, MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpExit, Register OpSizeReg, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI)
Adds logic in loop entry MBB to calculate loop iteration count and adds t2WhileLoopSetup and t2WhileL...
static SDValue createGPRPairNodei64(SelectionDAG &DAG, SDValue V)
static bool isLTorLE(ISD::CondCode CC)
static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl, SelectionDAG &DAG)
static SDValue performNegCMovCombine(SDNode *N, SelectionDAG &DAG)
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static TargetLowering::ArgListTy getDivRemArgList(const SDNode *N, FunctionType *FuncTy, const AttributeList &FuncAttrs, RTLIB::LibcallImpl LCImpl)
static SDValue PerformBITCASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG)
static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
static SDValue PerformInterleavedStoreCombine(StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG)
static bool hasNormalLoadOperand(SDNode *N)
hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node are normal,...
static SDValue PerformInsertEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
PerformInsertEltCombine - Target-specific dag combine xforms for ISD::INSERT_VECTOR_ELT.
static SDValue PerformVDUPLANECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVDUPLANECombine - Target-specific dag combine xforms for ARMISD::VDUPLANE.
static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static cl::opt< unsigned > ConstpoolPromotionMaxTotal("arm-promote-constant-max-total", cl::Hidden, cl::desc("Maximum size of ALL constants to promote into a constant pool"), cl::init(128))
static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static RTLIB::Libcall getDivRemLibcall(const SDNode *N, MVT::SimpleValueType SVT)
static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG &DAG)
SkipLoadExtensionForVMULL - return a load of the original vector size that does not do any sign/zero ...
static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombineWithOperands - Try DAG combinations for an ADD with operands N0 and N1.
static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT, SelectionDAG &DAG)
static SDValue matchCSET(unsigned &Opcode, bool &InvertCond, SDValue TrueVal, SDValue FalseVal, const ARMSubtarget *Subtarget)
static bool isSequentialConcatOfVectorInterleave(SDNode *N, SmallVectorImpl< SDValue > &Ops)
static SDValue PerformORCombineToSMULWBT(SDNode *OR, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue FindBFIToCombineWith(SDNode *N)
static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, bool &swpCmpOps, bool &swpVselOps)
static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isS16(const SDValue &Op, SelectionDAG &DAG)
static bool isSRA16(const SDValue &Op)
static SDValue AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue LowerInterruptReturn(SmallVectorImpl< SDValue > &RetOps, const SDLoc &DL, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue getInvertedARMCondCode(SDValue ARMcc, SelectionDAG &DAG)
static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl, SelectionDAG &DAG)
static void expandf64Toi32(SDValue Op, SelectionDAG &DAG, SDValue &RetVal1, SDValue &RetVal2)
static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isSHL16(const SDValue &Op)
static bool hasNativeAtomics(const ARMSubtarget &ST)
static bool isVEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseVEXT, unsigned &Imm)
static SDValue PerformMVEVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
cl::opt< unsigned > ArmMaxBaseUpdatesToCheck("arm-max-base-updates-to-check", cl::Hidden, cl::desc("Maximum number of base-updates to check generating postindex."), cl::init(64))
static bool isTruncMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue PerformADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2)
Return the load opcode for a given load size.
static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG, unsigned Opcode, bool IsSigned)
static bool isLegalT2AddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
static bool isLegalMVEShuffleOp(unsigned PFEntry)
static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N, SelectionDAG &DAG)
static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG)
PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for ISD::VECTOR_SHUFFLE.
static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG)
SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND, ANY_EXTEND,...
static int getNegationCost(SDValue Op)
static bool isVMOVNTruncMask(ArrayRef< int > M, EVT ToVT, bool rev)
static SDValue PerformVQMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static MachineBasicBlock * OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ)
static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformAddcSubcCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl)
getZeroVector - Returns a vector of specified type with all zero elements.
static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static ARMCC::CondCodes getVCMPCondCode(SDValue N)
static cl::opt< bool > ARMInterworking("arm-interworking", cl::Hidden, cl::desc("Enable / disable ARM interworking (for debugging only)"), cl::init(true))
static void ReplaceREADCYCLECOUNTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformORCombineToBFI(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, SDValue &CC, bool &Invert, SDValue &OtherOp, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVSetCCToVCTPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isZeroVector(SDValue N)
static SDValue PerformAddeSubeCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void ReplaceCMP_SWAP_64Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isLowerSaturate(const SDValue LHS, const SDValue RHS, const SDValue TrueVal, const SDValue FalseVal, const ISD::CondCode CC, const SDValue K)
static bool isLegalLogicalImmediate(unsigned Imm, const ARMSubtarget *Subtarget)
static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG)
static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned StSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment store operation with given size.
static bool isVMOVNMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue CombineBaseUpdate(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineBaseUpdate - Target-specific DAG combine function for VLDDUP, NEON load/store intrinsics,...
static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVRRDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMOVRRDCombine - Target-specific dag combine xforms for ARMISD::VMOVRRD.
static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain)
static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue PerformVMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMULCombine Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the special multi...
static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG)
static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformORCombine - Target-specific dag combine xforms for ISD::OR.
static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG)
static SDValue PerformTruncatingStoreCombine(StoreSDNode *St, SelectionDAG &DAG)
static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned LdSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment load operation with given size.
static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG)
static bool isValidBaseUpdate(SDNode *N, SDNode *User)
static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG, const ARMSubtarget *ST, const SDLoc &dl)
static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op)
static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformXORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment, bool isSEXTLoad, bool IsMasked, bool isLE, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
std::pair< unsigned, const TargetRegisterClass * > RCPair
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,...
static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
cl::opt< unsigned > MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden, cl::desc("Maximum interleave factor for MVE VLDn to generate."), cl::init(2))
static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, unsigned SplatBitSize, SelectionDAG &DAG, const SDLoc &dl, EVT &VT, EVT VectorVT, VMOVModImmType type)
isVMOVModifiedImm - Check if the specified splat value corresponds to a valid vector constant for a N...
static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC, SelectionDAG &DAG)
BC is a bitcast that is about to be turned into a VMOVDRR.
static SDValue promoteToConstantPool(const ARMTargetLowering *TLI, const GlobalValue *GV, SelectionDAG &DAG, EVT PtrVT, const SDLoc &dl)
static unsigned isNEONTwoResultShuffleMask(ArrayRef< int > ShuffleMask, EVT VT, unsigned &WhichResult, bool &isV_UNDEF)
Check if ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN), and return the corresponding AR...
static bool BitsProperlyConcatenate(const APInt &A, const APInt &B)
static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG)
static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target, struct BaseUpdateUser &User, bool SimpleConstIncOnly, TargetLowering::DAGCombinerInfo &DCI)
static bool allUsersAreInFunction(const Value *V, const Function *F)
Return true if all users of V are within function F, looking through ConstantExprs.
static bool isSingletonVEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG)
PerformVMOVDRRCombine - Target-specific dag combine xforms for ARMISD::VMOVDRR.
static SDValue PerformLegalizedInterleavedStoreCombine(StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, SDValue &SatK)
static bool isLegalAddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
isLegalAddressImmediate - Return true if the integer value can be used as the offset of the target ad...
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isLegalT1AddressImmediate(int64_t V, EVT VT)
static SDValue CombineANDShift(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSHLSimplify(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformADDECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDECombine - Target-specific dag combine transform from ARMISD::ADDC, ARMISD::ADDE,...
static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformHWLoopCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformORCombineToShiftInsert(SelectionDAG &DAG, SDValue AndOp, SDValue ShiftOp, EVT VT, SDLoc dl)
static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base, uint64_t &Members)
static SDValue PerformMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
static SDValue getShiftedLowMaskZeroTest(SDValue Value, SelectionDAG &DAG, const SDLoc &DL)
static SDValue PerformANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG)
static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm, bool &Negate)
static bool canChangeToInt(SDValue Op, bool &SeenZero, const ARMSubtarget *Subtarget)
canChangeToInt - Given the fp compare operand, return true if it is suitable to morph to an integer c...
static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2)
Return the store opcode for a given store size.
static bool IsVUZPShuffleNode(SDNode *N)
static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget, MachineInstr &MI, const SDNode *Node)
Attaches vregs to MEMCPY that it will use as scratch registers when it is expanded into LDM/STM.
static bool isFloatingPointZero(SDValue Op)
isFloatingPointZero - Return true if this is +0.0.
static SDValue findMUL_LOHI(SDValue V)
static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N, SelectionDAG &DAG)
static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG)
static void genTPLoopBody(MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI, Register OpSrcReg, Register OpDestReg, Register ElementCountReg, Register TotalIterationsReg, bool IsMemcpy)
Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and t2DoLoopEnd.
static SDValue PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformBUILD_VECTORCombine - Target-specific dag combine xforms for ISD::BUILD_VECTOR.
static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformMinMaxCombine - Target-specific DAG combining for creating truncating saturates.
arm ldst static false bool definesCPSR(const MachineInstr &MI)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis false
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static void createLoadIntrinsic(IntrinsicInst *II, LoadInst *LI, dxil::ResourceTypeInfo &RTI)
static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI, dxil::ResourceTypeInfo &RTI)
This file defines the DenseMap class.
static bool isSigned(unsigned Opcode)
const HexagonInstrInfo * TII
hexagon merge activate weight
Module.h This file contains the declarations for the Module class.
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
static Value * LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctpop of V before the specified instruction IP.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
Register const TargetRegisterInfo * TRI
Promote Memory to Register
uint64_t IntrinsicInst * II
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
Contains matchers for matching SelectionDAG nodes and values.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
This file describes how to lower LLVM code to machine code.
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Assuming the flags are set by MI(a,b), return the condition code if we modify the instructions such t...
static constexpr int Concat[]
static bool isIntrinsic(const CallBase &Call, Intrinsic::ID ID)
static constexpr roundingMode rmTowardZero
LLVM_ABI bool getExactInverse(APFloat *Inv) const
If this value is normal and has an exact, normal, multiplicative inverse, store it in inv and return ...
APInt bitcastToAPInt() const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
bool isMinSignedValue() const
Determine if this is the smallest signed value.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
LLVM_ABI APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static LLVM_ABI APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
bool isOne() const
Determine if this is a value of 1.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
const ARMBaseRegisterInfo & getRegisterInfo() const
const uint32_t * getSjLjDispatchPreservedMask(const MachineFunction &MF) const
const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const override
Code Generation virtual methods...
Register getFrameRegister(const MachineFunction &MF) const override
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
const uint32_t * getTLSCallPreservedMask(const MachineFunction &MF) const
const uint32_t * getThisReturnPreservedMask(const MachineFunction &MF, CallingConv::ID) const
getThisReturnPreservedMask - Returns a call preserved mask specific to the case that 'returned' is on...
static ARMConstantPoolConstant * Create(const Constant *C, unsigned ID)
static ARMConstantPoolMBB * Create(LLVMContext &C, const MachineBasicBlock *mbb, unsigned ID, unsigned char PCAdj)
static ARMConstantPoolSymbol * Create(LLVMContext &C, StringRef s, unsigned ID, unsigned char PCAdj, ARMCP::ARMCPModifier Modifier=ARMCP::no_modifier, bool AddCurrentAddress=false)
ARMConstantPoolValue - ARM specific constantpool value.
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
int getVarArgsFrameIndex() const
SmallVectorImpl< ForwardedRegister > & getForwardedMustTailRegParms()
int getPromotedConstpoolIncrease() const
SmallPtrSet< const GlobalVariable *, 2 > & getGlobalsPromotedToConstantPool()
void setArgumentStackToRestore(unsigned v)
bool branchTargetEnforcement() const
unsigned createPICLabelUId()
void setPromotedConstpoolIncrease(int Sz)
bool isThumb1OnlyFunction() const
void setArgRegsSaveSize(unsigned s)
bool isCmseNSEntryFunction() const
void setReturnRegsCount(unsigned s)
void setVarArgsFrameIndex(int Index)
unsigned getArgRegsSaveSize() const
void markGlobalAsPromotedToConstantPool(const GlobalVariable *GV)
Indicate to the backend that GV has had its storage changed to inside a constant pool.
void setIsSplitCSR(bool s)
void setArgumentStackSize(unsigned size)
unsigned getArgumentStackSize() const
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X, SDValue Y) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool isReadOnly(const GlobalValue *GV) const
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
bool shouldInsertFencesForAtomic(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const override
Return the correct alignment for the current calling convention.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
isLegalAddressingMode - Return true if the addressing mode represented by AM is legal for this target...
bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const
bool isLegalT1ScaledAddressingMode(const AddrMode &AM, EVT VT) const
Returns true if the addressing mode representing by AM is legal for the Thumb1 target,...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPreIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mod...
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
ReplaceNodeResults - Replace the results of node with an illegal result type with new values built ou...
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
bool isLegalAddImmediate(int64_t Imm) const override
isLegalAddImmediate - Return true if the specified immediate is legal add immediate,...
EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
SDValue PerformMVETruncCombine(SDNode *N, DAGCombinerInfo &DCI) const
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
FPImmAction getFPImmAction(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const override
Returns how to materialize the specified FP immediate.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
SDValue PerformIntrinsicCombine(SDNode *N, DAGCombinerInfo &DCI) const
PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
bool shouldFoldConstantShiftPairToMask(const SDNode *N) const override
Return true if it is profitable to fold a pair of shifts into a mask.
bool isDesirableToCommuteXorWithShift(const SDNode *N) const override
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const
PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const override
Create the IR node for the given complex deinterleaving operation.
bool isComplexDeinterleavingSupported() const override
Does this target support complex deinterleaving.
SDValue PerformMVEExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo, const LibcallLoweringInfo *libcallLowering) const override
createFastISel - This method returns a target specific FastISel object, or null if the target does no...
void insertSSPDeclarations(Module &M, const LibcallLoweringInfo &Libcalls) const override
Inserts necessary declarations for SSP (stack protection) purpose.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
getSetCCResultType - Return the value type to use for ISD::SETCC.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool isVarArg) const
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
isShuffleMaskLegal - Targets can use this to indicate that they only support some VECTOR_SHUFFLE oper...
Register getExceptionPointerRegister(ExceptionHandling EH, const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(const AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
bool lowerInterleavedStore(Instruction *Store, Value *Mask, ShuffleVectorInst *SVI, unsigned Factor, const APInt &GapMask) const override
Lower an interleaved store into a vstN intrinsic.
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const override
getRegClassFor - Return the register class that should be used for the specified value type.
bool useLoadStackGuardNode(const Module &M) const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
bool lowerInterleavedLoad(Instruction *Load, Value *Mask, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor, const APInt &GapMask) const override
Lower an interleaved load into a vldN intrinsic.
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool preferSelectsOverBooleanArithmetic(EVT VT) const override
Should we prefer selects to doing arithmetic on boolean types.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
Register getExceptionSelectorRegister(ExceptionHandling EH, const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return the cost of EXTRACT_SUBVECTOR for this result type with this index.
CallingConv::ID getEffectiveCallingConv(CallingConv::ID CC, bool isVarArg) const
getEffectiveCallingConv - Get the effective calling convention, taking into account presence of float...
ARMTargetLowering(const TargetMachine &TM, const ARMSubtarget &STI)
bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const override
Does this target support complex deinterleaving with the given operation and type.
bool supportKCFIBundles() const override
Return true if the target supports kcfi operand bundles.
SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const
PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
Value * getIRStackGuard(IRBuilderBase &IRB, const LibcallLoweringInfo &Libcalls) const override
If the target has a standard location for the stack protector guard, returns the address of that loca...
Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const override
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
Instruction * makeDMB(IRBuilderBase &Builder, ARM_MB::MemBOpt Domain) const
bool isLegalICmpImmediate(int64_t Imm) const override
isLegalICmpImmediate - Return true if the specified immediate is legal icmp immediate,...
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool isVarArg) const
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
allowsMisalignedMemoryAccesses - Returns true if the target allows unaligned memory accesses of the s...
bool isLegalInterleavedAccessType(unsigned Factor, FixedVectorType *VecTy, Align Alignment, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
bool isVectorLoadExtDesirable(SDValue ExtVal) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const override
Return true if the target can combine store(extractelement VectorTy,Idx).
bool useSoftFloat() const override
bool alignLoopsWithOptSize() const override
Should loops be aligned even when the function is marked OptSize (but not MinSize).
SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
LowerAsmOperandForConstraint - Lower the specified operand into the Ops vector.
bool hasAndNotCompare(SDValue V) const override
Return true if the target should transform: (X & Y) == Y ---> (~X & Y) == 0 (X & Y) !...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT.
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const override
Returns true if an argument of type Ty needs to be passed in a contiguous block of registers in calli...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
const ARMBaseTargetMachine & getTM() const
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPostIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mo...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
bool isFloatingPointOperation() const
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static LLVM_ABI BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, and fixed/scalable offsets.
LLVM Basic Block Representation.
The address of a basic block.
static constexpr BranchProbability getZero()
A "pseudo-class" with methods for operating on BUILD_VECTORs.
LLVM_ABI bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
LLVM_ABI int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
CCState - This class holds information needed while lowering arguments and return values.
void getInRegsParamInfo(unsigned InRegsParamRecordIndex, unsigned &BeginReg, unsigned &EndReg) const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
LLVM_ABI void analyzeMustTailForwardedRegisters(SmallVectorImpl< ForwardedRegister > &Forwards, ArrayRef< MVT > RegParmTypes, CCAssignFn Fn)
Compute the set of registers that need to be preserved and forwarded to any musttail calls.
static LLVM_ABI bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
LLVM_ABI bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
LLVM_ABI void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
void rewindByValRegsInfo()
unsigned getInRegsParamsProcessed() const
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void addInRegsParamInfo(unsigned RegBegin, unsigned RegEnd)
LLVM_ABI void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
unsigned getInRegsParamsCount() const
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
unsigned getValNo() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
AttributeList getAttributes() const
Return the attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
const APFloat & getValueAPF() const
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
MachineConstantPoolValue * getMachineCPVal() const
bool isMachineConstantPoolEntry() const
const Constant * getConstVal() const
LLVM_ABI Type * getType() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
MaybeAlign getStackAlignment() const
Returns the natural stack alignment, or MaybeAlign() if one wasn't specified.
LLVM_ABI TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
StringRef getInternalSymbolPrefix() const
LLVM_ABI Align getPreferredAlign(const GlobalVariable *GV) const
Returns the preferred alignment of the specified global.
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
iterator find(const_arg_type_t< KeyT > Val)
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
const Argument * const_arg_iterator
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
const GlobalValue * getGlobal() const
bool hasExternalWeakLinkage() const
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
bool isStrongDefinitionForLinker() const
Returns true if this global's definition will be the one chosen by the linker.
@ InternalLinkage
Rename collisions when linking (static functions).
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Module * getModule() const
Get the module.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
This is an important class for using LLVM in a threaded context.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
Tracks which library functions to use for a particular subtarget or function.
const RTLIB::RuntimeLibcallsInfo & getRuntimeLibcallsInfo() const
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Return the lowering's selection of implementation call for Call.
An instruction for reading from memory.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Describe properties that are true of each instruction in the target description file.
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static LLVM_ABI MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
bool is64BitVector() const
Return true if this is a 64-bit vector type.
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
bool isEHPad() const
Returns true if the block is a landing pad.
LLVM_ABI MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI bool canFallThrough()
Return true if the block can implicitly transfer control to the block after it by falling off the end...
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
LLVM_ABI MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
LLVM_ABI instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
iterator_range< pred_iterator > predecessors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
LLVM_ABI void moveAfter(MachineBasicBlock *NewBefore)
LLVM_ABI bool isLiveIn(MCRegister Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
LLVM_ABI unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
LLVM_ABI int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
LLVM_ABI void computeMaxCallFrameSize(MachineFunction &MF, std::vector< MachineBasicBlock::iterator > *FrameSDOps=nullptr)
Computes the maximum size of a callframe.
void setAdjustsStack(bool V)
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
bool hasMustTailInVarArgFunc() const
Returns true if the function is variadic and contains a musttail call.
void setReturnAddressIsTaken(bool s)
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool hasVAStart() const
Returns true if the function calls the llvm.va_start intrinsic.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
int getFunctionContextIndex() const
Return the index for the function context object.
Properties which a MachineFunction may have at a given point in time.
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
MachineOperand * mop_iterator
iterator/begin/end - Iterate over all operands of a machine instruction.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI unsigned createJumpTableIndex(const std::vector< MachineBasicBlock * > &DestBBs)
createJumpTableIndex - Create a new jump table.
@ EK_Inline
EK_Inline - Jump table entries are emitted inline at their point of use.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
LLVM_ABI void setIsRenamable(bool Val=true)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setIsDead(bool Val=true)
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
LLVM_ABI void setIsDef(bool Val=true)
Change a def to a use, or a use to a def.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
This class is used to represent an MLOAD node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const Triple & getTargetTriple() const
Get the target triple which is a string describing the target host.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
LLVM_ABI bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
ISD::CondCode getOperandCondCode(unsigned Num) const
Helper method returns the CondCode of a CondCodeSDNode.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
LLVM_ABI bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Returns true if the node type is UNDEF or POISON.
iterator_range< user_iterator > users()
void setFlags(SDNodeFlags NewFlags)
user_iterator user_begin() const
Provide iteration support to walk over all users of an SDNode.
op_iterator op_end() const
op_iterator op_begin() const
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
ISD::CondCode getOperandCondCode(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags, bool AllowCommute=false)
Get the specified node if it's already available, or else return NULL.
LLVM_ABI SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
LLVM_ABI bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, TypeSize Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
LLVM_ABI SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI bool areNonVolatileConsecutiveStores(StoreSDNode *ST, StoreSDNode *Base, TypeSize Bytes, int Dist) const
Return true if stores are next to each other and can be merged.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
LLVM_ABI MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI OverflowKind computeOverflowForSignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the signed addition of 2 nodes can overflow.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getRegisterMask(const uint32_t *RegMask)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
void addCallSiteInfo(const SDNode *Node, CallSiteInfo &&CallInfo)
Set CallSiteInfo to be associated with Node.
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
DenormalMode getDenormalMode(EVT VT) const
Return the current function's default denormal handling kind for the given floating point type.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
LLVM_ABI SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
static LLVM_ABI void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static LLVM_ABI bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
static LLVM_ABI bool isSplatMask(ArrayRef< int > Mask)
void insert_range(Range &&R)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
Represent a constant reference to a string, i.e.
const unsigned char * bytes_end() const
constexpr size_t size() const
Get the string size.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
const unsigned char * bytes_begin() const
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
void setMinStackArgumentAlignment(Align Alignment)
Set the minimum stack alignment of an argument.
virtual Value * getIRStackGuard(IRBuilderBase &IRB, const LibcallLoweringInfo &Libcalls) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
const TargetMachine & getTargetMachine() const
virtual void insertSSPDeclarations(Module &M, const LibcallLoweringInfo &Libcalls) const
Inserts necessary declarations for SSP (stack protection) purpose.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const
Returns true if the specified FP immediate can be materialized without a load from the constant pool.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
ExtractSubvectorCost
Enum that specifies how expensive lowering an EXTRACT_SUBVECTOR is.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
virtual unsigned getMaxSupportedInterleaveFactor() const
Get the maximum supported factor for interleaved memory accesses.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
FPImmAction
How the legalizer should materialize a floating-point immediate.
@ ConstantPool
Load the immediate from the constant pool.
@ Legal
The target can instruction select the immediate natively.
@ Bitcast
Build the immediate as a bitcast of an integer constant with the same bits.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Get the libcall impl routine name for the specified libcall.
static StringRef getLibcallImplName(RTLIB::LibcallImpl Call)
Get the libcall routine name for the specified libcall implementation.
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using an n/2-bit algorithm.
bool isPositionIndependent() const
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
static ArgListTy getArgListForFunctionType(FunctionType *FuncTy, const AttributeList &FuncAttrs, ArrayRef< SDValue > Ops)
Build a call argument list for FuncTy, taking the argument node values from Ops and the parameter typ...
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
TargetLowering(const TargetLowering &)=delete
virtual ArrayRef< MCPhysReg > getRoundingControlRegisters() const
Returns a 0 terminated array of rounding control registers that can be attached into strict FP call.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
void setTypeIdForCallsiteInfo(const CallBase *CB, MachineFunction &MF, MachineFunction::CallSiteInfo &CSInfo) const
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
bool isPointerTy() const
True if this is an instance of PointerType.
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
A Use represents the edge between a Value definition and its users.
LLVM_ABI unsigned getOperandNo() const
Return the operand # of this use in its User.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Base class of all SIMD vector types.
Type * getElementType() const
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static CondCodes getOppositeCondition(CondCodes CC)
static ARMCC::CondCodes getSwappedCondition(ARMCC::CondCodes CC)
getSwappedCondition - assume the flags are set by MI(a,b), return the condition code if we modify the...
@ SECREL
Thread Pointer Offset.
@ GOT_PREL
Thread Local Storage (General Dynamic Mode)
@ SBREL
Section Relative (Windows TLS)
@ GOTTPOFF
Global Offset Table, PC Relative.
@ TPOFF
Global Offset Table, Thread Pointer Offset.
TOF
Target Operand Flag enum.
@ MO_NONLAZY
MO_NONLAZY - This is an independent flag, on a symbol operand "FOO" it represents a symbol which,...
@ MO_SBREL
MO_SBREL - On a symbol operand, this represents a static base relative relocation.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_GOT
MO_GOT - On a symbol operand, this represents a GOT relative relocation.
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
unsigned getAM2Offset(unsigned AM2Opc)
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned createVMOVModImm(unsigned OpCmode, unsigned Val)
int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
int getFP32FP16Imm(const APInt &Imm)
If this is a FP16Imm encoded as a fp32 value, return the 8-bit encoding for it.
AddrOpc getAM2Op(unsigned AM2Opc)
bool isBitFieldInvertedMask(unsigned v)
const unsigned FPStatusBits
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo, const LibcallLoweringInfo *libcallLowering)
const unsigned FPReservedBits
const unsigned RoundingBitsPos
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Swift
Calling convention for Swift.
@ ARM_APCS
ARM Procedure Calling Standard (obsolete, but still used on some targets).
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ ARM_AAPCS
ARM Architecture Procedure Calling Standard calling convention (aka EABI).
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
@ C
The default llvm calling convention, compatible with C.
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ POISON
POISON - A poison node.
@ SET_FPENV
Sets the current floating-point environment.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ FMODF
FMODF - Decomposes the operand into integral and fractional parts, each having the same type and sign...
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ FSINCOSPI
FSINCOSPI - Compute both the sine and cosine times pi more accurately than FSINCOS(pi*x),...
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ CTLS
Count leading redundant sign bits.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
auto m_Value()
Match an arbitrary value and ignore it.
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
bool sd_match(SDValue N, Pattern &&P)
ConstantInt_match m_ConstInt()
Match any integer constants or splat of an integer constant.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
initializer< Ty > init(const Ty &Val)
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
void stable_sort(R &&Range)
bool isVTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of "vector_shuffle v,...
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns true if Val1 has a lower Constant Materialization Cost than Val2.
bool isVZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
RelativeUniformCounterPtr Values
@ Known
Known to have no common set bits.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Kill
The last use of a register.
@ Define
Register definition.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
bool CC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool RetCC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
bool RetCC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
LLVM_ABI Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
void shuffle(Iterator first, Iterator last, RNG &&g)
bool CC_ARM_APCS_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
static std::array< MachineOperand, 2 > predOps(ARMCC::CondCodes Pred, unsigned PredReg=0)
Get the operands corresponding to the given Pred value.
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
LLVM_ABI ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
auto dyn_cast_or_null(const Y &Val)
constexpr bool has_single_bit(T Value) noexcept
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
LLVM_ABI bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
bool FastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
ComplexDeinterleavingOperation
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool CC_ARM_Win32_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isVUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
bool isVTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
const unsigned PerfectShuffleTable[6561+1]
AtomicOrdering
Atomic ordering for LLVM's memory model.
ComplexDeinterleavingRotation
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
@ Fast
Assign the register banks as fast as possible (default).
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
DWARFExpression::Operation Op
bool isVUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of "vector_shuffle v,...
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr U AbsoluteValue(T X)
Return the absolute value of a signed integer, converted to the corresponding unsigned integer type.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
@ SjLj
setjmp/longjmp based exceptions
static MachineOperand t1CondCodeOp(bool isDead=false)
Get the operand corresponding to the conditional code result for Thumb1.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
UndefPoisonKind
Enumeration to track whether we are interested in Undef, Poison, or both.
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
static MachineOperand condCodeOp(unsigned CCReg=0)
Get the operand corresponding to the conditional code result.
bool isVREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isVREVMask - Check if a vector shuffle corresponds to a VREV instruction with the specified blocksize...
unsigned gettBLXrOpcode(const MachineFunction &MF)
bool CC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
@ Increment
Incrementally increasing token ID.
bool CC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
LLVM_ABI llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
constexpr bool isShiftedUInt(uint64_t x)
Checks if a unsigned integer is an N bit number shifted left by S.
unsigned convertAddSubFlagsOpcode(unsigned OldOpc)
Map pseudo instructions that imply an 'S' bit onto real opcodes.
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
bool isVZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of "vector_shuffle v,...
MCRegisterClass TargetRegisterClass
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Load/store instruction that can be merged with a base address update.
SDNode * N
Instruction that updates a pointer.
unsigned ConstInc
Pointer increment value if it is a constant, or 0 otherwise.
SDValue Inc
Pointer increment operand.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
static constexpr Align fromLog2(unsigned Log2Value)
Returns an alignment of 1 << Log2Value bytes.
static constexpr DenormalMode getIEEE()
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false, bool SelfAdd=false)
Compute knownbits resulting from addition of LHS and RHS.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
static LLVM_ABI KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
APInt getSignedMinValue() const
Return the minimal signed value possible given these KnownBits.
SmallVector< ArgRegPair, 1 > ArgRegPairs
Vector of call argument and its forwarding register.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static LLVM_ABI MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
A simple container for information about the supported runtime calls.
LLVM_ABI FunctionCallee getOrInsertLibcallDecl(Module &M, RTLIB::LibcallImpl LibcallImpl) const
LLVM_ABI GlobalValue * getOrInsertImplGlobal(Module &M, RTLIB::LibcallImpl LibcallImpl) const
LLVM_ABI std::pair< FunctionType *, AttributeList > getFunctionTy(LLVMContext &Ctx, const Triple &TT, const DataLayout &DL, RTLIB::LibcallImpl LibcallImpl) const
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setInRegister(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
bool isAfterLegalizeDAG() const
CombineLevel getDAGCombineLevel()
LLVM_ABI void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
LLVM_ABI bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
LLVM_ABI SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)