35#include "llvm/IR/IntrinsicsWebAssembly.h"
42#define DEBUG_TYPE "wasm-lower"
47 auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;
61 Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
67 if (Subtarget->hasSIMD128()) {
75 if (Subtarget->hasFP16()) {
78 if (Subtarget->hasReferenceTypes()) {
81 if (Subtarget->hasExceptionHandling()) {
90 for (
auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) {
94 if (Subtarget->hasSIMD128()) {
95 for (
auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
101 if (Subtarget->hasFP16()) {
105 if (Subtarget->hasReferenceTypes()) {
108 for (
auto T : {MVT::externref, MVT::funcref, MVT::Other}) {
129 for (
auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64, MVT::v8f16}) {
130 if (!Subtarget->hasFP16() &&
T == MVT::v8f16) {
145 for (
auto Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT,
146 ISD::FRINT, ISD::FROUNDEVEN})
153 if (
T != MVT::v8f16) {
166 for (
auto T : {MVT::i32, MVT::i64})
168 if (Subtarget->hasSIMD128())
169 for (
auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
173 if (Subtarget->hasWideArithmetic()) {
181 if (Subtarget->hasNontrappingFPToInt())
183 for (
auto T : {MVT::i32, MVT::i64})
187 if (Subtarget->hasSIMD128()) {
220 for (
auto T : {MVT::v16i8, MVT::v8i16})
224 for (
auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
228 for (
auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
232 if (Subtarget->hasFP16())
236 for (
auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
240 if (Subtarget->hasFP16())
244 for (
auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
252 for (
auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
257 for (
auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
265 for (
auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
272 for (
auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
277 for (
auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
287 for (
auto T : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
292 ISD::FEXP, ISD::FEXP2, ISD::FEXP10})
293 for (
auto T : {MVT::v4f32, MVT::v2f64})
303 for (
auto T : {MVT::v2i64, MVT::v2f64})
309 if (Subtarget->hasFP16()) {
321 for (
auto Op : {ISD::PARTIAL_REDUCE_SMLA, ISD::PARTIAL_REDUCE_UMLA}) {
330 if (!Subtarget->hasSignExt()) {
332 auto Action = Subtarget->hasSIMD128() ?
Custom :
Expand;
333 for (
auto T : {MVT::i8, MVT::i16, MVT::i32})
349 for (
auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
366 if (Subtarget->hasSIMD128()) {
367 for (
auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
370 if (
MVT(
T) != MemT) {
409 return MVT::externref;
418 return MVT::externref;
425WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(
AtomicRMWInst *AI)
const {
441bool WebAssemblyTargetLowering::shouldScalarizeBinop(
SDValue VecOp)
const {
461FastISel *WebAssemblyTargetLowering::createFastISel(
466MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(
const DataLayout & ,
477 "32-bit shift counts ought to be enough for anyone");
482 "Unable to represent scalar shift amount type");
492 bool IsUnsigned,
bool Int64,
493 bool Float64,
unsigned LoweredOpcode) {
499 unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
500 unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
501 unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
502 unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
503 unsigned IConst =
Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
504 unsigned Eqz = WebAssembly::EQZ_I32;
505 unsigned And = WebAssembly::AND_I32;
507 int64_t Substitute = IsUnsigned ? 0 : Limit;
508 double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
519 F->insert(It, FalseMBB);
520 F->insert(It, TrueMBB);
521 F->insert(It, DoneMBB);
524 DoneMBB->
splice(DoneMBB->
begin(), BB, std::next(
MI.getIterator()), BB->
end());
532 unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
533 Tmp0 =
MRI.createVirtualRegister(
MRI.getRegClass(InReg));
534 Tmp1 =
MRI.createVirtualRegister(
MRI.getRegClass(InReg));
535 CmpReg =
MRI.createVirtualRegister(&WebAssembly::I32RegClass);
536 EqzReg =
MRI.createVirtualRegister(&WebAssembly::I32RegClass);
537 FalseReg =
MRI.createVirtualRegister(
MRI.getRegClass(OutReg));
538 TrueReg =
MRI.createVirtualRegister(
MRI.getRegClass(OutReg));
540 MI.eraseFromParent();
554 Tmp1 =
MRI.createVirtualRegister(
MRI.getRegClass(InReg));
556 MRI.createVirtualRegister(&WebAssembly::I32RegClass);
557 Register AndReg =
MRI.createVirtualRegister(&WebAssembly::I32RegClass);
601 unsigned Eqz =
Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;
602 unsigned MemoryCopy =
603 Int64 ? WebAssembly::MEMORY_COPY_A64 : WebAssembly::MEMORY_COPY_A32;
614 F->insert(It, TrueMBB);
615 F->insert(It, DoneMBB);
618 DoneMBB->
splice(DoneMBB->
begin(), BB, std::next(
MI.getIterator()), BB->
end());
628 EqzReg =
MRI.createVirtualRegister(&WebAssembly::I32RegClass);
631 MI.eraseFromParent();
669 unsigned Eqz =
Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;
670 unsigned MemoryFill =
671 Int64 ? WebAssembly::MEMORY_FILL_A64 : WebAssembly::MEMORY_FILL_A32;
682 F->insert(It, TrueMBB);
683 F->insert(It, DoneMBB);
686 DoneMBB->
splice(DoneMBB->
begin(), BB, std::next(
MI.getIterator()), BB->
end());
696 EqzReg =
MRI.createVirtualRegister(&WebAssembly::I32RegClass);
699 MI.eraseFromParent();
721 CallResults.
getOpcode() == WebAssembly::RET_CALL_RESULTS);
725 bool IsRetCall = CallResults.
getOpcode() == WebAssembly::RET_CALL_RESULTS;
727 bool IsFuncrefCall =
false;
733 IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass);
738 if (IsIndirect && IsRetCall) {
739 CallOp = WebAssembly::RET_CALL_INDIRECT;
740 }
else if (IsIndirect) {
741 CallOp = WebAssembly::CALL_INDIRECT;
742 }
else if (IsRetCall) {
743 CallOp = WebAssembly::RET_CALL;
745 CallOp = WebAssembly::CALL;
774 for (
auto Def : CallResults.
defs())
798 for (
auto Use : CallParams.
uses())
814 if (IsIndirect && IsFuncrefCall) {
826 BuildMI(MF,
DL,
TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref);
830 BuildMI(MF,
DL,
TII.get(WebAssembly::TABLE_SET_FUNCREF))
842 const TargetInstrInfo &
TII = *Subtarget->getInstrInfo();
845 switch (
MI.getOpcode()) {
848 case WebAssembly::FP_TO_SINT_I32_F32:
850 WebAssembly::I32_TRUNC_S_F32);
851 case WebAssembly::FP_TO_UINT_I32_F32:
853 WebAssembly::I32_TRUNC_U_F32);
854 case WebAssembly::FP_TO_SINT_I64_F32:
856 WebAssembly::I64_TRUNC_S_F32);
857 case WebAssembly::FP_TO_UINT_I64_F32:
859 WebAssembly::I64_TRUNC_U_F32);
860 case WebAssembly::FP_TO_SINT_I32_F64:
862 WebAssembly::I32_TRUNC_S_F64);
863 case WebAssembly::FP_TO_UINT_I32_F64:
865 WebAssembly::I32_TRUNC_U_F64);
866 case WebAssembly::FP_TO_SINT_I64_F64:
868 WebAssembly::I64_TRUNC_S_F64);
869 case WebAssembly::FP_TO_UINT_I64_F64:
871 WebAssembly::I64_TRUNC_U_F64);
872 case WebAssembly::MEMCPY_A32:
874 case WebAssembly::MEMCPY_A64:
876 case WebAssembly::MEMSET_A32:
878 case WebAssembly::MEMSET_A64:
880 case WebAssembly::CALL_RESULTS:
881 case WebAssembly::RET_CALL_RESULTS:
887WebAssemblyTargetLowering::getTargetNodeName(
unsigned Opcode)
const {
891#define HANDLE_NODETYPE(NODE) \
892 case WebAssemblyISD::NODE: \
893 return "WebAssemblyISD::" #NODE;
894#include "WebAssemblyISD.def"
895#undef HANDLE_NODETYPE
900std::pair<unsigned, const TargetRegisterClass *>
901WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
905 if (Constraint.
size() == 1) {
906 switch (Constraint[0]) {
908 assert(VT != MVT::iPTR &&
"Pointer MVT not expected here");
909 if (Subtarget->hasSIMD128() && VT.
isVector()) {
911 return std::make_pair(0U, &WebAssembly::V128RegClass);
915 return std::make_pair(0U, &WebAssembly::I32RegClass);
917 return std::make_pair(0U, &WebAssembly::I64RegClass);
922 return std::make_pair(0U, &WebAssembly::F32RegClass);
924 return std::make_pair(0U, &WebAssembly::F64RegClass);
938bool WebAssemblyTargetLowering::isCheapToSpeculateCttz(
Type *Ty)
const {
943bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz(
Type *Ty)
const {
948bool WebAssemblyTargetLowering::isLegalAddressingMode(
const DataLayout &
DL,
950 Type *Ty,
unsigned AS,
966bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
980bool WebAssemblyTargetLowering::isIntDivCheap(
EVT VT,
981 AttributeList Attr)
const {
987bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(
SDValue ExtVal)
const {
990 return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
991 (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
992 (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
995bool WebAssemblyTargetLowering::isOffsetFoldingLegal(
1002EVT WebAssemblyTargetLowering::getSetCCResultType(
const DataLayout &
DL,
1015bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &
Info,
1019 switch (Intrinsic) {
1020 case Intrinsic::wasm_memory_atomic_notify:
1022 Info.memVT = MVT::i32;
1023 Info.ptrVal =
I.getArgOperand(0);
1034 case Intrinsic::wasm_memory_atomic_wait32:
1036 Info.memVT = MVT::i32;
1037 Info.ptrVal =
I.getArgOperand(0);
1042 case Intrinsic::wasm_memory_atomic_wait64:
1044 Info.memVT = MVT::i64;
1045 Info.ptrVal =
I.getArgOperand(0);
1050 case Intrinsic::wasm_loadf16_f32:
1052 Info.memVT = MVT::f16;
1053 Info.ptrVal =
I.getArgOperand(0);
1058 case Intrinsic::wasm_storef16_f32:
1060 Info.memVT = MVT::f16;
1061 Info.ptrVal =
I.getArgOperand(1);
1071void WebAssemblyTargetLowering::computeKnownBitsForTargetNode(
1074 switch (
Op.getOpcode()) {
1078 unsigned IntNo =
Op.getConstantOperandVal(0);
1082 case Intrinsic::wasm_bitmask: {
1084 EVT VT =
Op.getOperand(1).getSimpleValueType();
1087 Known.
Zero |= ZeroMask;
1097 case WebAssemblyISD::I64_ADD128:
1098 if (
Op.getResNo() == 1) {
1109WebAssemblyTargetLowering::getPreferredVectorAction(
MVT VT)
const {
1115 if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
1116 EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64)
1123bool WebAssemblyTargetLowering::shouldSimplifyDemandedVectorElts(
1124 SDValue Op,
const TargetLoweringOpt &TLO)
const {
1177WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
1179 SelectionDAG &DAG = CLI.DAG;
1189 "WebAssembly doesn't support language-specific or target-specific "
1190 "calling conventions yet");
1191 if (CLI.IsPatchPoint)
1192 fail(
DL, DAG,
"WebAssembly doesn't support patch point yet");
1194 if (CLI.IsTailCall) {
1195 auto NoTail = [&](
const char *Msg) {
1196 if (CLI.CB && CLI.CB->isMustTailCall())
1198 CLI.IsTailCall =
false;
1201 if (!Subtarget->hasTailCall())
1202 NoTail(
"WebAssembly 'tail-call' feature not enabled");
1206 NoTail(
"WebAssembly does not support varargs tail calls");
1211 Type *RetTy =
F.getReturnType();
1216 bool TypesMatch = CallerRetTys.
size() == CalleeRetTys.
size() &&
1217 std::equal(CallerRetTys.
begin(), CallerRetTys.
end(),
1218 CalleeRetTys.
begin());
1220 NoTail(
"WebAssembly tail call requires caller and callee return types to "
1225 for (
auto &Arg : CLI.CB->args()) {
1226 Value *Val = Arg.get();
1231 Src =
GEP->getPointerOperand();
1238 "WebAssembly does not support tail calling with stack arguments");
1245 SmallVectorImpl<ISD::InputArg> &
Ins = CLI.Ins;
1246 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1247 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1253 Outs[0].Flags.isSRet()) {
1258 bool HasSwiftSelfArg =
false;
1259 bool HasSwiftErrorArg =
false;
1260 unsigned NumFixedArgs = 0;
1261 for (
unsigned I = 0;
I < Outs.
size(); ++
I) {
1262 const ISD::OutputArg &Out = Outs[
I];
1267 fail(
DL, DAG,
"WebAssembly hasn't implemented nest arguments");
1269 fail(
DL, DAG,
"WebAssembly hasn't implemented inalloca arguments");
1271 fail(
DL, DAG,
"WebAssembly hasn't implemented cons regs arguments");
1273 fail(
DL, DAG,
"WebAssembly hasn't implemented cons regs last arguments");
1282 Chain = DAG.
getMemcpy(Chain,
DL, FINode, OutVal, SizeNode,
1285 nullptr, std::nullopt, MachinePointerInfo(),
1286 MachinePointerInfo());
1293 bool IsVarArg = CLI.IsVarArg;
1302 if (!HasSwiftSelfArg) {
1304 ISD::ArgFlagsTy
Flags;
1305 Flags.setSwiftSelf();
1306 ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
1307 CLI.Outs.push_back(Arg);
1309 CLI.OutVals.push_back(ArgVal);
1311 if (!HasSwiftErrorArg) {
1313 ISD::ArgFlagsTy
Flags;
1314 Flags.setSwiftError();
1315 ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
1316 CLI.Outs.push_back(Arg);
1318 CLI.OutVals.push_back(ArgVal);
1324 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.
getContext());
1329 for (
unsigned I = NumFixedArgs;
I < Outs.
size(); ++
I) {
1330 const ISD::OutputArg &Out = Outs[
I];
1333 assert(VT != MVT::iPTR &&
"Legalized args should be concrete");
1338 CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment);
1345 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
1348 if (IsVarArg && NumBytes) {
1351 MaybeAlign StackAlign = Layout.getStackAlignment();
1352 assert(StackAlign &&
"data layout string is missing stack alignment");
1358 assert(ArgLocs[ValNo].getValNo() == ValNo &&
1359 "ArgLocs should remain in order and only hold varargs args");
1360 unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
1368 if (!Chains.
empty())
1370 }
else if (IsVarArg) {
1388 Ops.push_back(Chain);
1389 Ops.push_back(Callee);
1394 IsVarArg ? OutVals.
begin() + NumFixedArgs : OutVals.
end());
1397 Ops.push_back(FINode);
1400 for (
const auto &In : Ins) {
1401 assert(!
In.Flags.isByVal() &&
"byval is not valid for return values");
1402 assert(!
In.Flags.isNest() &&
"nest is not valid for return values");
1403 if (
In.Flags.isInAlloca())
1404 fail(
DL, DAG,
"WebAssembly hasn't implemented inalloca return values");
1405 if (
In.Flags.isInConsecutiveRegs())
1406 fail(
DL, DAG,
"WebAssembly hasn't implemented cons regs return values");
1407 if (
In.Flags.isInConsecutiveRegsLast())
1409 "WebAssembly hasn't implemented cons regs last return values");
1418 CLI.CB->getCalledOperand()->getType())) {
1433 WebAssemblyISD::TABLE_SET,
DL, DAG.
getVTList(MVT::Other), TableSetOps,
1438 CLI.CB->getCalledOperand()->getPointerAlignment(DAG.
getDataLayout()),
1444 if (CLI.IsTailCall) {
1446 SDVTList NodeTys = DAG.
getVTList(MVT::Other, MVT::Glue);
1447 return DAG.
getNode(WebAssemblyISD::RET_CALL,
DL, NodeTys,
Ops);
1451 SDVTList InTyList = DAG.
getVTList(InTys);
1454 for (
size_t I = 0;
I <
Ins.size(); ++
I)
1461bool WebAssemblyTargetLowering::CanLowerReturn(
1464 const Type *RetTy)
const {
1469SDValue WebAssemblyTargetLowering::LowerReturn(
1475 "MVP WebAssembly can only return up to one value");
1477 fail(
DL, DAG,
"WebAssembly doesn't support non-C calling conventions");
1480 RetOps.append(OutVals.
begin(), OutVals.
end());
1481 Chain = DAG.
getNode(WebAssemblyISD::RETURN,
DL, MVT::Other, RetOps);
1484 for (
const ISD::OutputArg &Out : Outs) {
1489 fail(
DL, DAG,
"WebAssembly hasn't implemented inalloca results");
1491 fail(
DL, DAG,
"WebAssembly hasn't implemented cons regs results");
1493 fail(
DL, DAG,
"WebAssembly hasn't implemented cons regs last results");
1499SDValue WebAssemblyTargetLowering::LowerFormalArguments(
1504 fail(
DL, DAG,
"WebAssembly doesn't support non-C calling conventions");
1507 auto *MFI = MF.
getInfo<WebAssemblyFunctionInfo>();
1513 bool HasSwiftErrorArg =
false;
1514 bool HasSwiftSelfArg =
false;
1515 for (
const ISD::InputArg &In : Ins) {
1516 HasSwiftSelfArg |=
In.Flags.isSwiftSelf();
1517 HasSwiftErrorArg |=
In.Flags.isSwiftError();
1518 if (
In.Flags.isInAlloca())
1519 fail(
DL, DAG,
"WebAssembly hasn't implemented inalloca arguments");
1520 if (
In.Flags.isNest())
1521 fail(
DL, DAG,
"WebAssembly hasn't implemented nest arguments");
1522 if (
In.Flags.isInConsecutiveRegs())
1523 fail(
DL, DAG,
"WebAssembly hasn't implemented cons regs arguments");
1524 if (
In.Flags.isInConsecutiveRegsLast())
1525 fail(
DL, DAG,
"WebAssembly hasn't implemented cons regs last arguments");
1534 MFI->addParam(
In.VT);
1543 if (!HasSwiftSelfArg) {
1544 MFI->addParam(PtrVT);
1546 if (!HasSwiftErrorArg) {
1547 MFI->addParam(PtrVT);
1556 MFI->setVarargBufferVreg(VarargVreg);
1558 Chain,
DL, VarargVreg,
1559 DAG.
getNode(WebAssemblyISD::ARGUMENT,
DL, PtrVT,
1561 MFI->addParam(PtrVT);
1573 assert(MFI->getParams().size() == Params.
size() &&
1574 std::equal(MFI->getParams().begin(), MFI->getParams().end(),
1580void WebAssemblyTargetLowering::ReplaceNodeResults(
1582 switch (
N->getOpcode()) {
1596 Results.push_back(Replace128Op(
N, DAG));
1600 "ReplaceNodeResults not implemented for this op for WebAssembly!");
1611 switch (
Op.getOpcode()) {
1616 return LowerFrameIndex(
Op, DAG);
1618 return LowerGlobalAddress(
Op, DAG);
1620 return LowerGlobalTLSAddress(
Op, DAG);
1622 return LowerExternalSymbol(
Op, DAG);
1624 return LowerJumpTable(
Op, DAG);
1626 return LowerBR_JT(
Op, DAG);
1628 return LowerVASTART(
Op, DAG);
1631 fail(
DL, DAG,
"WebAssembly hasn't implemented computed gotos");
1634 return LowerRETURNADDR(
Op, DAG);
1636 return LowerFRAMEADDR(
Op, DAG);
1638 return LowerCopyToReg(
Op, DAG);
1641 return LowerAccessVectorElement(
Op, DAG);
1645 return LowerIntrinsic(
Op, DAG);
1647 return LowerSIGN_EXTEND_INREG(
Op, DAG);
1650 return LowerEXTEND_VECTOR_INREG(
Op, DAG);
1652 return LowerBUILD_VECTOR(
Op, DAG);
1654 return LowerVECTOR_SHUFFLE(
Op, DAG);
1656 return LowerSETCC(
Op, DAG);
1660 return LowerShift(
Op, DAG);
1663 return LowerFP_TO_INT_SAT(
Op, DAG);
1665 return LowerLoad(
Op, DAG);
1667 return LowerStore(
Op, DAG);
1676 return LowerMUL_LOHI(
Op, DAG);
1678 return LowerUADDO(
Op, DAG);
1693 return std::nullopt;
1712 SDVTList Tys = DAG.
getVTList(MVT::Other);
1724 SDVTList Tys = DAG.
getVTList(MVT::Other);
1726 return DAG.
getNode(WebAssemblyISD::LOCAL_SET,
DL, Tys,
Ops);
1731 "Encountered an unlowerable store to the wasm_var address space",
1747 "unexpected offset when loading from webassembly global",
false);
1758 "unexpected offset when loading from webassembly local",
false);
1765 assert(
Result->getNumValues() == 2 &&
"Loads must carry a chain!");
1771 "Encountered an unlowerable load from the wasm_var address space",
1779 assert(Subtarget->hasWideArithmetic());
1780 assert(
Op.getValueType() == MVT::i64);
1783 switch (
Op.getOpcode()) {
1785 Opcode = WebAssemblyISD::I64_MUL_WIDE_U;
1788 Opcode = WebAssemblyISD::I64_MUL_WIDE_S;
1809 assert(Subtarget->hasWideArithmetic());
1810 assert(
Op.getValueType() == MVT::i64);
1817 DAG.
getNode(WebAssemblyISD::I64_ADD128,
DL,
1827 assert(Subtarget->hasWideArithmetic());
1828 assert(
N->getValueType(0) == MVT::i128);
1831 switch (
N->getOpcode()) {
1833 Opcode = WebAssemblyISD::I64_ADD128;
1836 Opcode = WebAssemblyISD::I64_SUB128;
1851 LHS_0, LHS_1, RHS_0, RHS_1);
1868 EVT VT = Src.getValueType();
1870 : WebAssembly::COPY_I64,
1873 return Op.getNode()->getNumValues() == 1
1892 if (!Subtarget->getTargetTriple().isOSEmscripten()) {
1894 "Non-Emscripten WebAssembly hasn't implemented "
1895 "__builtin_return_address");
1899 unsigned Depth =
Op.getConstantOperandVal(0);
1901 return makeLibCall(DAG, RTLIB::RETURN_ADDRESS,
Op.getValueType(),
1902 {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions,
DL)
1911 if (
Op.getConstantOperandVal(0) > 0)
1915 EVT VT =
Op.getValueType();
1922WebAssemblyTargetLowering::LowerGlobalTLSAddress(
SDValue Op,
1928 if (!MF.
getSubtarget<WebAssemblySubtarget>().hasBulkMemory())
1932 const GlobalValue *GV = GA->
getGlobal();
1937 auto model = Subtarget->getTargetTriple().isOSEmscripten()
1952 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
1953 : WebAssembly::GLOBAL_GET_I32;
1964 DAG.
getNode(WebAssemblyISD::WrapperREL,
DL, PtrVT, TLSOffset);
1971 EVT VT =
Op.getValueType();
1972 return DAG.
getNode(WebAssemblyISD::Wrapper,
DL, VT,
1982 EVT VT =
Op.getValueType();
1984 "Unexpected target flags on generic GlobalAddressSDNode");
1986 fail(
DL, DAG,
"Invalid address space for WebAssembly target");
1989 const GlobalValue *GV = GA->
getGlobal();
1997 const char *BaseName;
2006 DAG.
getNode(WebAssemblyISD::Wrapper,
DL, PtrVT,
2010 WebAssemblyISD::WrapperREL,
DL, VT,
2019 return DAG.
getNode(WebAssemblyISD::Wrapper,
DL, VT,
2025WebAssemblyTargetLowering::LowerExternalSymbol(
SDValue Op,
2029 EVT VT =
Op.getValueType();
2030 assert(ES->getTargetFlags() == 0 &&
2031 "Unexpected target flags on generic ExternalSymbolSDNode");
2032 return DAG.
getNode(WebAssemblyISD::Wrapper,
DL, VT,
2043 JT->getTargetFlags());
2052 assert(
JT->getTargetFlags() == 0 &&
"WebAssembly doesn't set target flags");
2055 Ops.push_back(Chain);
2056 Ops.push_back(Index);
2062 for (
auto *
MBB : MBBs)
2069 return DAG.
getNode(WebAssemblyISD::BR_TABLE,
DL, MVT::Other,
Ops);
2081 MFI->getVarargBufferVreg(), PtrVT);
2082 return DAG.
getStore(
Op.getOperand(0),
DL, ArgN,
Op.getOperand(1),
2083 MachinePointerInfo(SV));
2090 switch (
Op.getOpcode()) {
2093 IntNo =
Op.getConstantOperandVal(1);
2096 IntNo =
Op.getConstantOperandVal(0);
2107 case Intrinsic::wasm_lsda: {
2116 DAG.
getNode(WebAssemblyISD::Wrapper,
DL, PtrVT,
2119 DAG.
getNode(WebAssemblyISD::WrapperREL,
DL, PtrVT, Node);
2123 return DAG.
getNode(WebAssemblyISD::Wrapper,
DL, PtrVT, Node);
2126 case Intrinsic::wasm_shuffle: {
2132 while (
OpIdx < 18) {
2141 return DAG.
getNode(WebAssemblyISD::SHUFFLE,
DL,
Op.getValueType(),
Ops);
2144 case Intrinsic::thread_pointer: {
2146 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
2147 : WebAssembly::GLOBAL_GET_I32;
2158WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(
SDValue Op,
2168 assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128());
2172 const SDValue &Extract =
Op.getOperand(0);
2176 MVT ExtractedLaneT =
2180 if (ExtractedVecT == VecT)
2187 unsigned IndexVal =
Index->getAsZExtVal();
2205 assert((UserOpc == WebAssemblyISD::EXTEND_LOW_U ||
2206 UserOpc == WebAssemblyISD::EXTEND_LOW_S) &&
2207 "expected extend_low");
2212 size_t FirstIdx = Mask.size() / 2;
2213 for (
size_t i = 0; i < Mask.size() / 2; ++i) {
2214 if (Mask[i] !=
static_cast<int>(FirstIdx + i)) {
2220 unsigned Opc = UserOpc == WebAssemblyISD::EXTEND_LOW_S
2221 ? WebAssemblyISD::EXTEND_HIGH_S
2222 : WebAssemblyISD::EXTEND_HIGH_U;
2223 return DAG.
getNode(
Opc,
DL, VT, Shuffle->getOperand(0));
2227WebAssemblyTargetLowering::LowerEXTEND_VECTOR_INREG(
SDValue Op,
2230 EVT VT =
Op.getValueType();
2232 EVT SrcVT = Src.getValueType();
2239 "Unexpected extension factor.");
2242 if (Scale != 2 && Scale != 4 && Scale != 8)
2246 switch (
Op.getOpcode()) {
2248 Ext = WebAssemblyISD::EXTEND_LOW_U;
2251 Ext = WebAssemblyISD::EXTEND_LOW_S;
2262 while (Scale != 1) {
2265 .widenIntegerVectorElementType(*DAG.
getContext())
2266 .getHalfNumVectorElementsVT(*DAG.
getContext()),
2276 if (
Op.getValueType() != MVT::v2f64)
2280 unsigned &Index) ->
bool {
2281 switch (
Op.getOpcode()) {
2283 Opcode = WebAssemblyISD::CONVERT_LOW_S;
2286 Opcode = WebAssemblyISD::CONVERT_LOW_U;
2288 case ISD::FP_EXTEND:
2289 Opcode = WebAssemblyISD::PROMOTE_LOW;
2295 auto ExtractVector =
Op.getOperand(0);
2302 SrcVec = ExtractVector.getOperand(0);
2303 Index = ExtractVector.getConstantOperandVal(1);
2307 unsigned LHSOpcode, RHSOpcode, LHSIndex, RHSIndex;
2309 if (!GetConvertedLane(
Op.getOperand(0), LHSOpcode, LHSSrcVec, LHSIndex) ||
2310 !GetConvertedLane(
Op.getOperand(1), RHSOpcode, RHSSrcVec, RHSIndex))
2313 if (LHSOpcode != RHSOpcode)
2317 switch (LHSOpcode) {
2318 case WebAssemblyISD::CONVERT_LOW_S:
2319 case WebAssemblyISD::CONVERT_LOW_U:
2320 ExpectedSrcVT = MVT::v4i32;
2322 case WebAssemblyISD::PROMOTE_LOW:
2323 ExpectedSrcVT = MVT::v4f32;
2329 auto Src = LHSSrcVec;
2330 if (LHSIndex != 0 || RHSIndex != 1 || LHSSrcVec != RHSSrcVec) {
2333 ExpectedSrcVT,
DL, LHSSrcVec, RHSSrcVec,
2334 {
static_cast<int>(LHSIndex),
static_cast<int>(RHSIndex) + 4, -1, -1});
2336 return DAG.
getNode(LHSOpcode,
DL, MVT::v2f64, Src);
2341 MVT VT =
Op.getSimpleValueType();
2342 if (VT == MVT::v8f16) {
2357 const EVT VecT =
Op.getValueType();
2358 const EVT LaneT =
Op.getOperand(0).getValueType();
2360 bool CanSwizzle = VecT == MVT::v16i8;
2381 auto GetSwizzleSrcs = [](
size_t I,
const SDValue &Lane) {
2385 const SDValue &SwizzleSrc = Lane->getOperand(0);
2386 const SDValue &IndexExt = Lane->getOperand(1);
2396 Index->getConstantOperandVal(1) !=
I)
2398 return std::make_pair(SwizzleSrc, SwizzleIndices);
2405 auto GetShuffleSrc = [&](
const SDValue &Lane) {
2410 if (Lane->getOperand(0).getValueType().getVectorNumElements() >
2413 return Lane->getOperand(0);
2416 using ValueEntry = std::pair<SDValue, size_t>;
2419 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>,
size_t>;
2422 using ShuffleEntry = std::pair<SDValue, size_t>;
2425 auto AddCount = [](
auto &Counts,
const auto &Val) {
2428 if (CountIt == Counts.end()) {
2429 Counts.emplace_back(Val, 1);
2435 auto GetMostCommon = [](
auto &Counts) {
2437 assert(CommonIt != Counts.end() &&
"Unexpected all-undef build_vector");
2441 size_t NumConstantLanes = 0;
2444 for (
size_t I = 0;
I < Lanes; ++
I) {
2449 AddCount(SplatValueCounts, Lane);
2453 if (
auto ShuffleSrc = GetShuffleSrc(Lane))
2454 AddCount(ShuffleCounts, ShuffleSrc);
2456 auto SwizzleSrcs = GetSwizzleSrcs(
I, Lane);
2457 if (SwizzleSrcs.first)
2458 AddCount(SwizzleCounts, SwizzleSrcs);
2463 size_t NumSplatLanes;
2464 std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);
2468 size_t NumSwizzleLanes = 0;
2469 if (SwizzleCounts.
size())
2470 std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),
2471 NumSwizzleLanes) = GetMostCommon(SwizzleCounts);
2475 SDValue ShuffleSrc1, ShuffleSrc2;
2476 size_t NumShuffleLanes = 0;
2477 if (ShuffleCounts.
size()) {
2478 std::tie(ShuffleSrc1, NumShuffleLanes) = GetMostCommon(ShuffleCounts);
2480 [&](
const auto &Pair) {
return Pair.first == ShuffleSrc1; });
2482 if (ShuffleCounts.
size()) {
2483 size_t AdditionalShuffleLanes;
2484 std::tie(ShuffleSrc2, AdditionalShuffleLanes) =
2485 GetMostCommon(ShuffleCounts);
2486 NumShuffleLanes += AdditionalShuffleLanes;
2491 std::function<bool(
size_t,
const SDValue &)> IsLaneConstructed;
2494 if (NumSwizzleLanes >= NumShuffleLanes &&
2495 NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) {
2498 auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
2499 IsLaneConstructed = [&, Swizzled](
size_t I,
const SDValue &Lane) {
2500 return Swizzled == GetSwizzleSrcs(
I, Lane);
2502 }
else if (NumShuffleLanes >= NumConstantLanes &&
2503 NumShuffleLanes >= NumSplatLanes) {
2513 assert(LaneSize > DestLaneSize);
2514 Scale1 = LaneSize / DestLaneSize;
2520 assert(LaneSize > DestLaneSize);
2521 Scale2 = LaneSize / DestLaneSize;
2526 assert(DestLaneCount <= 16);
2527 for (
size_t I = 0;
I < DestLaneCount; ++
I) {
2529 SDValue Src = GetShuffleSrc(Lane);
2530 if (Src == ShuffleSrc1) {
2532 }
else if (Src && Src == ShuffleSrc2) {
2538 ArrayRef<int> MaskRef(Mask, DestLaneCount);
2540 IsLaneConstructed = [&](size_t,
const SDValue &Lane) {
2541 auto Src = GetShuffleSrc(Lane);
2542 return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2);
2544 }
else if (NumConstantLanes >= NumSplatLanes) {
2546 for (
const SDValue &Lane :
Op->op_values()) {
2555 int64_t Val =
Const ?
Const->getSExtValue() : 0;
2556 uint64_t LaneBits = 128 / Lanes;
2557 assert((LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) &&
2558 "Unexpected out of bounds negative value");
2559 if (Const && LaneBits != 64 && Val > (1ll << (LaneBits - 1)) - 1) {
2560 uint64_t
Mask = (1ll << LaneBits) - 1;
2561 auto NewVal = (((uint64_t)Val &
Mask) - (1ll << LaneBits)) & Mask;
2578 if (NumSplatLanes == 1 &&
Op->getOperand(0) == SplatValue &&
2579 (DestLaneSize == 32 || DestLaneSize == 64)) {
2586 IsLaneConstructed = [&SplatValue](
size_t _,
const SDValue &Lane) {
2587 return Lane == SplatValue;
2592 assert(IsLaneConstructed);
2595 for (
size_t I = 0;
I < Lanes; ++
I) {
2597 if (!Lane.
isUndef() && !IsLaneConstructed(
I, Lane))
2606WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(
SDValue Op,
2610 MVT VecType =
Op.getOperand(0).getSimpleValueType();
2621 for (
int M : Mask) {
2622 for (
size_t J = 0; J < LaneBytes; ++J) {
2626 uint64_t ByteIndex =
M == -1 ? J : (uint64_t)M * LaneBytes + J;
2631 return DAG.
getNode(WebAssemblyISD::SHUFFLE,
DL,
Op.getValueType(),
Ops);
2639 assert(
Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64);
2644 auto MakeLane = [&](
unsigned I) {
2650 {MakeLane(0), MakeLane(1)});
2654WebAssemblyTargetLowering::LowerAccessVectorElement(
SDValue Op,
2671 EVT LaneT =
Op.getSimpleValueType().getVectorElementType();
2673 if (LaneT.
bitsGE(MVT::i32))
2677 size_t NumLanes =
Op.getSimpleValueType().getVectorNumElements();
2679 unsigned ShiftOpcode =
Op.getOpcode();
2685 for (
size_t i = 0; i < NumLanes; ++i) {
2688 SDValue ShiftedValue = ShiftedElements[i];
2693 DAG.
getNode(ShiftOpcode,
DL, MVT::i32, ShiftedValue, MaskedShiftValue));
2703 assert(
Op.getSimpleValueType().isVector());
2705 uint64_t LaneBits =
Op.getValueType().getScalarSizeInBits();
2706 auto ShiftVal =
Op.getOperand(1);
2709 auto SkipImpliedMask = [](
SDValue MaskOp, uint64_t MaskBits) {
2720 MaskVal == MaskBits)
2727 if (ConstantRHS && ConstantRHS->getAPIntValue() == MaskBits)
2735 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
2741 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
2746 switch (
Op.getOpcode()) {
2748 Opcode = WebAssemblyISD::VEC_SHL;
2751 Opcode = WebAssemblyISD::VEC_SHR_S;
2754 Opcode = WebAssemblyISD::VEC_SHR_U;
2760 return DAG.
getNode(Opcode,
DL,
Op.getValueType(),
Op.getOperand(0), ShiftVal);
2765 EVT ResT =
Op.getValueType();
2768 if ((ResT == MVT::i32 || ResT == MVT::i64) &&
2769 (SatVT == MVT::i32 || SatVT == MVT::i64))
2772 if (ResT == MVT::v4i32 && SatVT == MVT::i32)
2775 if (ResT == MVT::v8i16 && SatVT == MVT::i16)
2786 auto &DAG = DCI.
DAG;
2793 SDValue Bitcast =
N->getOperand(0);
2794 if (Bitcast.getOpcode() != ISD::BITCAST)
2796 if (!
N->getOperand(1).isUndef())
2798 SDValue CastOp = Bitcast.getOperand(0);
2800 EVT DstType = Bitcast.getValueType();
2801 if (!SrcType.is128BitVector() ||
2802 SrcType.getVectorNumElements() != DstType.getVectorNumElements())
2805 SrcType,
SDLoc(
N), CastOp, DAG.
getUNDEF(SrcType), Shuffle->getMask());
2815 auto &DAG = DCI.
DAG;
2819 EVT InVT =
N->getOperand(0)->getValueType(0);
2820 EVT ResVT =
N->getValueType(0);
2822 if (ResVT == MVT::v4f32 && (InVT == MVT::v4i16 || InVT == MVT::v4i8))
2824 else if (ResVT == MVT::v2f64 && (InVT == MVT::v2i16 || InVT == MVT::v2i8))
2838 auto &DAG = DCI.
DAG;
2842 EVT VT =
N->getValueType(0);
2856 auto &DAG = DCI.
DAG;
2862 auto Extract =
N->getOperand(0);
2867 if (IndexNode ==
nullptr)
2869 auto Index = IndexNode->getZExtValue();
2873 EVT ResVT =
N->getValueType(0);
2874 if (ResVT == MVT::v8i16) {
2876 Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))
2878 }
else if (ResVT == MVT::v4i32) {
2880 Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))
2882 }
else if (ResVT == MVT::v2i64) {
2884 Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2))
2891 bool IsLow = Index == 0;
2893 unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S
2894 : WebAssemblyISD::EXTEND_HIGH_S)
2895 : (IsLow ? WebAssemblyISD::EXTEND_LOW_U
2896 : WebAssemblyISD::EXTEND_HIGH_U);
2903 auto &DAG = DCI.
DAG;
2905 auto GetWasmConversionOp = [](
unsigned Op) {
2908 return WebAssemblyISD::TRUNC_SAT_ZERO_S;
2910 return WebAssemblyISD::TRUNC_SAT_ZERO_U;
2912 return WebAssemblyISD::DEMOTE_ZERO;
2917 auto IsZeroSplat = [](
SDValue SplatVal) {
2919 APInt SplatValue, SplatUndef;
2920 unsigned SplatBitSize;
2925 Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
2943 EVT ExpectedConversionType;
2946 switch (ConversionOp) {
2950 ExpectedConversionType = MVT::v2i32;
2954 ExpectedConversionType = MVT::v2f32;
2960 if (
N->getValueType(0) != ResVT)
2963 if (
Conversion.getValueType() != ExpectedConversionType)
2967 if (Source.getValueType() != MVT::v2f64)
2970 if (!IsZeroSplat(
N->getOperand(1)) ||
2971 N->getOperand(1).getValueType() != ExpectedConversionType)
2974 unsigned Op = GetWasmConversionOp(ConversionOp);
2990 auto ConversionOp =
N->getOpcode();
2991 switch (ConversionOp) {
3003 if (
N->getValueType(0) != ResVT)
3006 auto Concat =
N->getOperand(0);
3007 if (
Concat.getValueType() != MVT::v4f64)
3010 auto Source =
Concat.getOperand(0);
3011 if (Source.getValueType() != MVT::v2f64)
3014 if (!IsZeroSplat(
Concat.getOperand(1)) ||
3015 Concat.getOperand(1).getValueType() != MVT::v2f64)
3018 unsigned Op = GetWasmConversionOp(ConversionOp);
3024 const SDLoc &
DL,
unsigned VectorWidth) {
3032 unsigned ElemsPerChunk = VectorWidth / ElVT.
getSizeInBits();
3037 IdxVal &= ~(ElemsPerChunk - 1);
3042 Vec->
ops().slice(IdxVal, ElemsPerChunk));
3054 EVT SrcVT = In.getValueType();
3072 EVT InVT = MVT::i16, OutVT = MVT::i8;
3077 unsigned SubSizeInBits = SrcSizeInBits / 2;
3079 OutVT =
EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits());
3105 auto &DAG = DCI.
DAG;
3108 EVT InVT = In.getValueType();
3112 EVT OutVT =
N->getValueType(0);
3119 if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) &&
3120 (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.
is128BitVector()))
3133 auto &DAG = DCI.
DAG;
3136 EVT VT =
N->getValueType(0);
3137 EVT SrcVT = Src.getValueType();
3148 if (NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) {
3151 {DAG.getConstant(Intrinsic::wasm_bitmask, DL, MVT::i32),
3152 DAG.getSExtOrTrunc(N->getOperand(0), DL,
3153 SrcVT.changeVectorElementType(Width))}),
3158 if (NumElts == 32 || NumElts == 64) {
3184 MVT ReturnType = VectorsToShuffle.
size() == 2 ? MVT::i32 : MVT::i64;
3187 for (
SDValue V : VectorsToShuffle) {
3188 ReturningInteger = DAG.
getNode(
3197 return ReturningInteger;
3215 EVT LT =
LHS.getValueType();
3216 if (LT.getScalarSizeInBits() > 128 / LT.getVectorNumElements())
3219 auto CombineSetCC = [&
N, &DAG](Intrinsic::WASMIntrinsics InPre,
3221 Intrinsic::WASMIntrinsics InPost) {
3222 if (
N->getConstantOperandVal(0) != InPre)
3233 {DAG.getConstant(InPost, DL, MVT::i32), LHS}),
3236 Ret = DAG.
getNOT(
DL, Ret, MVT::i1);
3241 Intrinsic::wasm_alltrue))
3244 Intrinsic::wasm_anytrue))
3247 Intrinsic::wasm_anytrue))
3250 Intrinsic::wasm_alltrue))
3256template <
int MatchRHS,
ISD::CondCode MatchCond,
bool RequiresNegate,
3271 {DAG.getConstant(Intrin, DL, MVT::i32),
3272 DAG.getSExtOrTrunc(LHS->getOperand(0), DL, VecVT)}),
3275 Ret = DAG.
getNOT(
DL, Ret, MVT::i1);
3288 EVT VT =
N->getValueType(0);
3289 EVT OpVT =
X.getValueType();
3293 Attribute::NoImplicitFloat))
3299 !Subtarget->
hasSIMD128() || !isIntEqualitySetCC(CC))
3303 auto IsVectorBitCastCheap = [](
SDValue X) {
3308 if (!IsVectorBitCastCheap(
X) || !IsVectorBitCastCheap(
Y))
3318 : Intrinsic::wasm_anytrue,
3332 EVT VT =
N->getValueType(0);
3340 if (
LHS->getOpcode() != ISD::BITCAST)
3343 EVT FromVT =
LHS->getOperand(0).getValueType();
3348 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
3355 auto &DAG = DCI.
DAG;
3384 EVT VT =
N->getValueType(0);
3385 if (VT != MVT::v8i32 && VT != MVT::v16i32)
3391 if (
LHS.getOpcode() !=
RHS.getOpcode())
3398 if (
LHS->getOperand(0).getValueType() !=
RHS->getOperand(0).getValueType())
3401 EVT FromVT =
LHS->getOperand(0).getValueType();
3403 if (EltTy != MVT::i8)
3431 unsigned ExtendLowOpc =
3432 IsSigned ? WebAssemblyISD::EXTEND_LOW_S : WebAssemblyISD::EXTEND_LOW_U;
3433 unsigned ExtendHighOpc =
3434 IsSigned ? WebAssemblyISD::EXTEND_HIGH_S : WebAssemblyISD::EXTEND_HIGH_U;
3436 auto GetExtendLow = [&DAG, &
DL, &ExtendLowOpc](
EVT VT,
SDValue Op) {
3443 if (NumElts == 16) {
3444 SDValue LowLHS = GetExtendLow(MVT::v8i16, ExtendInLHS);
3445 SDValue LowRHS = GetExtendLow(MVT::v8i16, ExtendInRHS);
3451 GetExtendLow(MVT::v4i32, MulLow),
3453 GetExtendLow(MVT::v4i32, MulHigh),
3462 SDValue Lo = GetExtendLow(MVT::v4i32, MulLow);
3472 EVT VT =
N->getValueType(0);
3489 EVT MulVT = MVT::v8i16;
3491 if (VT == MVT::v8i8) {
3497 DAG.
getNode(WebAssemblyISD::EXTEND_LOW_U,
DL, MulVT, PromotedLHS);
3499 DAG.
getNode(WebAssemblyISD::EXTEND_LOW_U,
DL, MulVT, PromotedRHS);
3504 MVT::v16i8,
DL, MulLow, DAG.
getUNDEF(MVT::v16i8),
3505 {0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1});
3508 assert(VT == MVT::v16i8 &&
"Expected v16i8");
3512 DAG.
getNode(WebAssemblyISD::EXTEND_HIGH_U,
DL, MulVT,
LHS);
3514 DAG.
getNode(WebAssemblyISD::EXTEND_HIGH_U,
DL, MulVT,
RHS);
3523 VT,
DL, MulLow, MulHigh,
3524 {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30});
3529WebAssemblyTargetLowering::PerformDAGCombine(
SDNode *
N,
3530 DAGCombinerInfo &DCI)
const {
3531 switch (
N->getOpcode()) {
unsigned const MachineRegisterInfo * MRI
static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue performTruncateCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis false
Function Alias Analysis Results
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg, SDValue Val={})
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
const HexagonInstrInfo * TII
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Register const TargetRegisterInfo * TRI
Promote Memory to Register
MachineInstr unsigned OpIdx
static SDValue performVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try to map an integer comparison with size > XLEN to vector instructions before type legalization spl...
const SmallVectorImpl< MachineOperand > & Cond
Contains matchers for matching SelectionDAG nodes and values.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static MachineBasicBlock * LowerFPToInt(MachineInstr &MI, DebugLoc DL, MachineBasicBlock *BB, const TargetInstrInfo &TII, bool IsUnsigned, bool Int64, bool Float64, unsigned LoweredOpcode)
static bool callingConvSupported(CallingConv::ID CallConv)
static SDValue TryWideExtMulCombine(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * LowerMemcpy(MachineInstr &MI, DebugLoc DL, MachineBasicBlock *BB, const TargetInstrInfo &TII, bool Int64)
static std::optional< unsigned > IsWebAssemblyLocal(SDValue Op, SelectionDAG &DAG)
static SDValue performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performVectorNonNegToFPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG)
static SDValue performAnyAllCombine(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB, const WebAssemblySubtarget *Subtarget, const TargetInstrInfo &TII)
static SDValue TryMatchTrue(SDNode *N, EVT VecVT, SelectionDAG &DAG)
static SDValue GetExtendHigh(SDValue Op, unsigned UserOpc, EVT VT, SelectionDAG &DAG)
static SDValue performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool IsWebAssemblyGlobal(SDValue Op)
static MachineBasicBlock * LowerMemset(MachineInstr &MI, DebugLoc DL, MachineBasicBlock *BB, const TargetInstrInfo &TII, bool Int64)
static SDValue performVectorExtendToFPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get split up into scalar instr...
static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG)
static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &DL, unsigned VectorWidth)
static SDValue performBitcastCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG)
This file defines the interfaces that WebAssembly uses to lower LLVM code into a selection DAG.
This file provides WebAssembly-specific target descriptions.
This file declares WebAssembly-specific per-machine-function information.
This file declares the WebAssembly-specific subclass of TargetSubtarget.
This file declares the WebAssembly-specific subclass of TargetMachine.
This file contains the declaration of the WebAssembly-specific type parsing utility functions.
This file contains the declaration of the WebAssembly-specific utility functions.
static constexpr int Concat[]
Class for arbitrary precision integers.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
an instruction that atomically reads a memory location, combines it with another value,...
BinOp getOperation() const
LLVM Basic Block Representation.
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
This class represents a function call, abstracting a target machine's calling convention.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
Diagnostic information for unsupported feature in backend.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
FunctionType * getFunctionType() const
Returns the FunctionType for me.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
int64_t getOffset() const
LLVM_ABI unsigned getAddressSpace() const
unsigned getTargetFlags() const
const GlobalValue * getGlobal() const
ThreadLocalMode getThreadLocalMode() const
Type * getValueType() const
This is an important class for using LLVM in a threaded context.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
Describe properties that are true of each instruction in the target description file.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
@ INVALID_SIMPLE_VALUE_TYPE
static auto integer_fixedlen_vector_valuetypes()
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
bool isFixedLengthVector() const
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
const char * createExternalSymbolName(StringRef Name)
Allocate a string and populate it with the given external symbol name.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
mop_range defs()
Returns all explicit operands that are register definitions.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
LLVM_ABI void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
mop_range uses()
Returns all operands which may be register uses.
LLVM_ABI void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
LLVM_ABI void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
const MachineOperand & getOperand(unsigned i) const
const std::vector< MachineJumpTableEntry > & getJumpTables() const
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setIsKill(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
LLVM_ABI SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
const DataLayout & getDataLayout() const
SDValue getTargetFrameIndex(int FI, EVT VT)
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
LLVM_ABI SDValue getBasicBlock(MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
LLVM_ABI SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SDValue & getValue() const
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setMinimumJumpTableEntries(unsigned Val)
Indicate the minimum number of blocks to generate jump tables.
void setPartialReduceMLAAction(unsigned Opc, MVT AccVT, MVT InputVT, LegalizeAction Action)
Indicate how a PARTIAL_REDUCE_U/SMLA node with Acc type AccVT and Input type InputVT should be treate...
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
TargetLowering(const TargetLowering &)=delete
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
Primary interface to the complete machine description for the target machine.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isFunctionTy() const
True if this is an instance of FunctionType.
static LLVM_ABI Type * getDoubleTy(LLVMContext &C)
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
LLVM_ABI const Value * stripPointerCastsAndAliases() const
Strip off pointer casts, all-zero GEPs, address space casts, and aliases.
static std::optional< unsigned > getLocalForStackObject(MachineFunction &MF, int FrameIndex)
bool hasCallIndirectOverlong() const
bool hasReferenceTypes() const
WebAssemblyTargetLowering(const TargetMachine &TM, const WebAssemblySubtarget &STI)
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const override
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const override
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Swift
Calling convention for Swift.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ CXX_FAST_TLS
Used for access functions.
@ WASM_EmscriptenInvoke
For emscripten __invoke_* functions.
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ BSWAP
Byte Swap and Counting operators.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
LLVM_ABI bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
OperandFlags
These are flags set on operands, but should be considered private, all access should go through the M...
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
bool sd_match(SDNode *N, const SelectionDAG *DAG, Pattern &&P)
CondCode_match m_SpecificCondCode(ISD::CondCode CC)
Match a conditional code SDNode with a specific ISD::CondCode.
CondCode_match m_CondCode()
Match any conditional code SDNode.
TernaryOpc_match< T0_P, T1_P, T2_P, true, false > m_c_SetCC(const T0_P &LHS, const T1_P &RHS, const T2_P &CC)
MCSymbolWasm * getOrCreateFunctionTableSymbol(MCContext &Ctx, const WebAssemblySubtarget *Subtarget)
Returns the __indirect_function_table, for use in call_indirect and in function bitcasts.
@ WASM_ADDRESS_SPACE_EXTERNREF
@ WASM_ADDRESS_SPACE_FUNCREF
bool isWebAssemblyFuncrefType(const Type *Ty)
Return true if this is a WebAssembly Funcref Type.
bool isWebAssemblyTableType(const Type *Ty)
Return true if the table represents a WebAssembly table type.
MCSymbolWasm * getOrCreateFuncrefCallTableSymbol(MCContext &Ctx, const WebAssemblySubtarget *Subtarget)
Returns the __funcref_call_table, for use in funcref calls when lowered to table.set + call_indirect.
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
bool isValidAddressSpace(unsigned AS)
bool canLowerReturn(size_t ResultSize, const WebAssemblySubtarget *Subtarget)
Returns true if the function's return value(s) can be lowered directly, i.e., not indirectly via a po...
bool isWasmVarAddressSpace(unsigned AS)
NodeAddr< NodeBase * > Node
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
FunctionAddr VTableAddr Value
void computeSignatureVTs(const FunctionType *Ty, const Function *TargetFunc, const Function &ContextFunc, const TargetMachine &TM, SmallVectorImpl< MVT > &Params, SmallVectorImpl< MVT > &Results)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
DWARFExpression::Operation Op
auto max_element(R &&Range)
Provide wrappers to std::max_element which take ranges instead of having to pass begin/end explicitly...
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
void computeLegalValueVTs(const WebAssemblyTargetLowering &TLI, LLVMContext &Ctx, const DataLayout &DL, Type *Ty, SmallVectorImpl< MVT > &ValueVTs)
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInConsecutiveRegs() const
Align getNonZeroOrigAlign() const
bool isSwiftError() const
unsigned getByValSize() const
bool isInConsecutiveRegsLast() const
Align getNonZeroByValAlign() const
unsigned getBitWidth() const
Get the bit width of this value.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
bool isBeforeLegalize() const
This structure is used to pass arguments to makeLibCall function.