26#include "llvm/IR/IntrinsicsS390.h"
35#define DEBUG_TYPE "systemz-lower"
41 : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
42 Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
92 if (Subtarget.hasHighWord())
98 if (Subtarget.hasVector()) {
105 if (Subtarget.hasVectorEnhancements1())
110 if (Subtarget.hasVector()) {
119 if (Subtarget.hasVector())
146 for (
unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
147 I <= MVT::LAST_FP_VALUETYPE;
173 for (
unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
174 I <= MVT::LAST_INTEGER_VALUETYPE;
205 if (Subtarget.hasPopulationCount())
224 if (!Subtarget.hasFPExtension())
230 if (Subtarget.hasFPExtension())
235 if (Subtarget.hasFPExtension())
304 {MVT::i8, MVT::i16, MVT::i32},
Legal);
306 {MVT::i8, MVT::i16},
Legal);
323 if (!Subtarget.hasFPExtension()) {
336 if (Subtarget.hasMiscellaneousExtensions3()) {
432 if (VT != MVT::v2i64)
438 if (Subtarget.hasVectorEnhancements1())
469 if (Subtarget.hasVector()) {
491 if (Subtarget.hasVectorEnhancements2()) {
512 for (
unsigned I = MVT::FIRST_FP_VALUETYPE;
513 I <= MVT::LAST_FP_VALUETYPE;
521 if (Subtarget.hasFPExtension()) {
549 if (Subtarget.hasFPExtension()) {
560 if (Subtarget.hasVector()) {
606 if (Subtarget.hasVectorEnhancements1()) {
613 if (Subtarget.hasVectorEnhancements1()) {
667 for (
auto VT : { MVT::f32, MVT::f64, MVT::f128,
668 MVT::v4f32, MVT::v2f64 }) {
677 if (!Subtarget.hasVectorEnhancements1()) {
683 if (Subtarget.hasVectorEnhancements1())
693 if (Subtarget.hasVectorEnhancements1()) {
705 if (!Subtarget.hasVector()) {
760 struct RTLibCallMapping {
764 static RTLibCallMapping RTLibCallCommon[] = {
765#define HANDLE_LIBCALL(code, name) {RTLIB::code, name},
766#include "ZOSLibcallNames.def"
768 for (
auto &E : RTLibCallCommon)
774 return Subtarget.hasSoftFloat();
796 return Subtarget.hasVectorEnhancements1();
809 if (!Subtarget.hasVector() ||
810 (isFP128 && !Subtarget.hasVectorEnhancements1()))
832 if (SplatBitSize > 64)
838 if (isInt<16>(SignedValue)) {
847 if (
TII->isRxSBGMask(
Value, SplatBitSize, Start,
End)) {
869 uint64_t Lower = SplatUndefZ & maskTrailingOnes<uint64_t>(LowerBits);
870 uint64_t Upper = SplatUndefZ & maskLeadingOnes<uint64_t>(UpperBits);
877 uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
878 return tryValue(SplatBitsZ | Middle);
893 unsigned HalfSize = Width / 2;
898 if (HighValue != LowValue || 8 > HalfSize)
901 SplatBits = HighValue;
905 SplatBitSize = Width;
913 BVN->
isConstantSplat(IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, 128,
917 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, 8,
922 bool ForCodeSize)
const {
924 if (Imm.isZero() || Imm.isNegZero())
956 if (Subtarget.hasInterlockedAccess1() &&
970 return isInt<32>(Imm) || isUInt<32>(Imm);
975 return isUInt<32>(Imm) || isUInt<32>(-Imm);
997 LongDisplacement(LongDispl), IndexReg(IdxReg) {}
1020 switch (
II->getIntrinsicID()) {
1022 case Intrinsic::memset:
1023 case Intrinsic::memmove:
1024 case Intrinsic::memcpy:
1029 if (isa<LoadInst>(
I) &&
I->hasOneUse()) {
1030 auto *SingleUser = cast<Instruction>(*
I->user_begin());
1031 if (SingleUser->getParent() ==
I->getParent()) {
1032 if (isa<ICmpInst>(SingleUser)) {
1033 if (
auto *
C = dyn_cast<ConstantInt>(SingleUser->getOperand(1)))
1034 if (
C->getBitWidth() <= 64 &&
1035 (isInt<16>(
C->getSExtValue()) || isUInt<16>(
C->getZExtValue())))
1038 }
else if (isa<StoreInst>(SingleUser))
1042 }
else if (
auto *StoreI = dyn_cast<StoreInst>(
I)) {
1043 if (
auto *LoadI = dyn_cast<LoadInst>(StoreI->getValueOperand()))
1044 if (LoadI->hasOneUse() && LoadI->getParent() ==
I->getParent())
1049 if (HasVector && (isa<LoadInst>(
I) || isa<StoreInst>(
I))) {
1057 Type *MemAccessTy = (isa<LoadInst>(
I) ?
I->getType() :
1058 I->getOperand(0)->getType());
1060 bool IsVectorAccess = MemAccessTy->
isVectorTy();
1064 if (!IsVectorAccess && isa<StoreInst>(
I)) {
1065 Value *DataOp =
I->getOperand(0);
1066 if (isa<ExtractElementInst>(DataOp))
1067 IsVectorAccess =
true;
1072 if (!IsVectorAccess && isa<LoadInst>(
I) &&
I->hasOneUse()) {
1073 User *LoadUser = *
I->user_begin();
1074 if (isa<InsertElementInst>(LoadUser))
1075 IsVectorAccess =
true;
1078 if (IsFPAccess || IsVectorAccess)
1107 return AM.
Scale == 0;
1114 std::vector<EVT> &MemOps,
unsigned Limit,
const MemOp &
Op,
unsigned DstAS,
1115 unsigned SrcAS,
const AttributeList &FuncAttributes)
const {
1116 const int MVCFastLen = 16;
1118 if (Limit != ~
unsigned(0)) {
1120 if (
Op.isMemcpy() &&
Op.allowOverlap() &&
Op.size() <= MVCFastLen)
1122 if (
Op.isMemset() &&
Op.size() - 1 <= MVCFastLen)
1124 if (
Op.isZeroMemset())
1129 SrcAS, FuncAttributes);
1134 return Subtarget.hasVector() ? MVT::v2i64 : MVT::Other;
1138 if (!FromType->isIntegerTy() || !ToType->
isIntegerTy())
1140 unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();
1142 return FromBits > ToBits;
1150 return FromBits > ToBits;
1159 if (Constraint.
size() == 1) {
1160 switch (Constraint[0]) {
1186 }
else if (Constraint.
size() == 2 && Constraint[0] ==
'Z') {
1187 switch (Constraint[1]) {
1203 const char *constraint)
const {
1205 Value *CallOperandVal =
info.CallOperandVal;
1208 if (!CallOperandVal)
1212 switch (*constraint) {
1230 if (Subtarget.hasVector())
1236 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1237 if (isUInt<8>(
C->getZExtValue()))
1242 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1243 if (isUInt<12>(
C->getZExtValue()))
1248 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1249 if (isInt<16>(
C->getSExtValue()))
1254 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1255 if (isInt<20>(
C->getSExtValue()))
1260 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1261 if (
C->getZExtValue() == 0x7fffffff)
1271static std::pair<unsigned, const TargetRegisterClass *>
1273 const unsigned *Map,
unsigned Size) {
1274 assert(*(Constraint.
end()-1) ==
'}' &&
"Missing '}'");
1275 if (isdigit(Constraint[2])) {
1280 return std::make_pair(Map[
Index], RC);
1282 return std::make_pair(0U,
nullptr);
1285std::pair<unsigned, const TargetRegisterClass *>
1288 if (Constraint.
size() == 1) {
1290 switch (Constraint[0]) {
1295 return std::make_pair(0U, &SystemZ::GR64BitRegClass);
1297 return std::make_pair(0U, &SystemZ::GR128BitRegClass);
1298 return std::make_pair(0U, &SystemZ::GR32BitRegClass);
1302 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass);
1303 else if (VT == MVT::i128)
1304 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass);
1305 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass);
1308 return std::make_pair(0U, &SystemZ::GRH32BitRegClass);
1313 return std::make_pair(0U, &SystemZ::FP64BitRegClass);
1315 return std::make_pair(0U, &SystemZ::FP128BitRegClass);
1316 return std::make_pair(0U, &SystemZ::FP32BitRegClass);
1321 if (Subtarget.hasVector()) {
1323 return std::make_pair(0U, &SystemZ::VR32BitRegClass);
1325 return std::make_pair(0U, &SystemZ::VR64BitRegClass);
1326 return std::make_pair(0U, &SystemZ::VR128BitRegClass);
1335 auto getVTSizeInBits = [&VT]() {
1343 if (Constraint[1] ==
'r') {
1344 if (getVTSizeInBits() == 32)
1347 if (getVTSizeInBits() == 128)
1353 if (Constraint[1] ==
'f') {
1355 return std::make_pair(
1357 if (getVTSizeInBits() == 32)
1360 if (getVTSizeInBits() == 128)
1366 if (Constraint[1] ==
'v') {
1367 if (!Subtarget.hasVector())
1368 return std::make_pair(
1370 if (getVTSizeInBits() == 32)
1373 if (getVTSizeInBits() == 64)
1400 const Constant *PersonalityFn)
const {
1405 const Constant *PersonalityFn)
const {
1413 if (Constraint.
size() == 1) {
1414 switch (Constraint[0]) {
1416 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op))
1417 if (isUInt<8>(
C->getZExtValue()))
1419 Op.getValueType()));
1423 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op))
1424 if (isUInt<12>(
C->getZExtValue()))
1426 Op.getValueType()));
1430 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op))
1431 if (isInt<16>(
C->getSExtValue()))
1433 Op.getValueType()));
1437 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op))
1438 if (isInt<20>(
C->getSExtValue()))
1440 Op.getValueType()));
1444 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op))
1445 if (
C->getZExtValue() == 0x7fffffff)
1447 Op.getValueType()));
1458#include "SystemZGenCallingConv.inc"
1462 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1468 Type *ToType)
const {
1531 if (BitCastToType == MVT::v2i64)
1558 MVT::Untyped,
Hi,
Lo);
1582 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
1584 if (ValueVT.
getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
1595 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
1596 if (ValueVT.
getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
1623 unsigned NumFixedGPRs = 0;
1624 unsigned NumFixedFPRs = 0;
1625 for (
unsigned I = 0, E = ArgLocs.
size();
I != E; ++
I) {
1638 RC = &SystemZ::GR32BitRegClass;
1642 RC = &SystemZ::GR64BitRegClass;
1646 RC = &SystemZ::FP32BitRegClass;
1650 RC = &SystemZ::FP64BitRegClass;
1654 RC = &SystemZ::FP128BitRegClass;
1662 RC = &SystemZ::VR128BitRegClass;
1691 ArgValue = DAG.
getLoad(LocVT,
DL, Chain, FIN,
1702 unsigned ArgIndex = Ins[
I].OrigArgIndex;
1703 assert (Ins[
I].PartOffset == 0);
1704 while (
I + 1 != E && Ins[
I + 1].OrigArgIndex == ArgIndex) {
1706 unsigned PartOffset = Ins[
I + 1].PartOffset;
1729 int64_t VarArgOffset = CCInfo.
getStackSize() + Regs->getCallFrameSize();
1747 int64_t RegSaveOffset =
1762 &SystemZ::FP64BitRegClass);
1780 MRI.addLiveIn(Regs->getADARegister(), ADAvReg);
1792 for (
unsigned I = 0, E = ArgLocs.
size();
I != E; ++
I) {
1799 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
1801 if (Outs[
I].Flags.isSwiftSelf() || Outs[
I].Flags.isSwiftError())
1808 unsigned Offset,
bool LoadAdr =
false) {
1831 bool LoadAddr =
false;
1832 const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV);
1853 unsigned ADADelta = 0;
1854 unsigned EPADelta = 8;
1859 if (
auto *
G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1860 bool IsInternal = (
G->getGlobal()->hasInternalLinkage() ||
1861 G->getGlobal()->hasPrivateLinkage());
1876 }
else if (
auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
1938 for (
unsigned I = 0, E = ArgLocs.
size();
I != E; ++
I) {
1944 unsigned ArgIndex = Outs[
I].OrigArgIndex;
1946 if (
I + 1 != E && Outs[
I + 1].OrigArgIndex == ArgIndex) {
1948 Type *OrigArgType = CLI.
Args[Outs[
I].OrigArgIndex].Ty;
1954 SlotVT = Outs[
I].VT;
1957 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
1963 assert (Outs[
I].PartOffset == 0);
1964 while (
I + 1 != E && Outs[
I + 1].OrigArgIndex == ArgIndex) {
1965 SDValue PartValue = OutVals[
I + 1];
1966 unsigned PartOffset = Outs[
I + 1].PartOffset;
1973 SlotVT.
getStoreSize()) &&
"Not enough space for argument part!");
1976 ArgValue = SpillSlot;
1993 if (!StackPtr.getNode())
2015 RegsToPass.
push_back(std::make_pair(SystemZ::R3D, ShadowArgValue));
2021 if (!MemOpChains.
empty())
2034 ->getAddressOfCalleeRegister();
2037 Callee = DAG.
getRegister(CalleeReg, Callee.getValueType());
2042 if (
auto *
G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2045 }
else if (
auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
2048 }
else if (IsTailCall) {
2051 Callee = DAG.
getRegister(SystemZ::R1D, Callee.getValueType());
2056 for (
unsigned I = 0, E = RegsToPass.
size();
I != E; ++
I) {
2058 RegsToPass[
I].second, Glue);
2069 for (
unsigned I = 0, E = RegsToPass.
size();
I != E; ++
I)
2071 RegsToPass[
I].second.getValueType()));
2075 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
2076 assert(Mask &&
"Missing call preserved mask for calling convention");
2100 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
2104 for (
unsigned I = 0, E = RetLocs.
size();
I != E; ++
I) {
2126 bool DoesNotReturn,
bool IsReturnValueUsed)
const {
2128 Args.reserve(Ops.
size());
2133 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.
getContext());
2136 Args.push_back(Entry);
2162 for (
auto &Out : Outs)
2163 if (Out.ArgVT == MVT::i128)
2167 CCState RetCCInfo(CallConv, isVarArg, MF, RetLocs, Context);
2168 return RetCCInfo.
CheckReturn(Outs, RetCC_SystemZ);
2185 if (RetLocs.
empty())
2195 for (
unsigned I = 0, E = RetLocs.
size();
I != E; ++
I) {
2224 unsigned &CCValid) {
2225 unsigned Id =
Op.getConstantOperandVal(1);
2227 case Intrinsic::s390_tbegin:
2232 case Intrinsic::s390_tbegin_nofloat:
2237 case Intrinsic::s390_tend:
2251 unsigned Id =
Op.getConstantOperandVal(0);
2253 case Intrinsic::s390_vpkshs:
2254 case Intrinsic::s390_vpksfs:
2255 case Intrinsic::s390_vpksgs:
2260 case Intrinsic::s390_vpklshs:
2261 case Intrinsic::s390_vpklsfs:
2262 case Intrinsic::s390_vpklsgs:
2267 case Intrinsic::s390_vceqbs:
2268 case Intrinsic::s390_vceqhs:
2269 case Intrinsic::s390_vceqfs:
2270 case Intrinsic::s390_vceqgs:
2275 case Intrinsic::s390_vchbs:
2276 case Intrinsic::s390_vchhs:
2277 case Intrinsic::s390_vchfs:
2278 case Intrinsic::s390_vchgs:
2283 case Intrinsic::s390_vchlbs:
2284 case Intrinsic::s390_vchlhs:
2285 case Intrinsic::s390_vchlfs:
2286 case Intrinsic::s390_vchlgs:
2291 case Intrinsic::s390_vtm:
2296 case Intrinsic::s390_vfaebs:
2297 case Intrinsic::s390_vfaehs:
2298 case Intrinsic::s390_vfaefs:
2303 case Intrinsic::s390_vfaezbs:
2304 case Intrinsic::s390_vfaezhs:
2305 case Intrinsic::s390_vfaezfs:
2310 case Intrinsic::s390_vfeebs:
2311 case Intrinsic::s390_vfeehs:
2312 case Intrinsic::s390_vfeefs:
2317 case Intrinsic::s390_vfeezbs:
2318 case Intrinsic::s390_vfeezhs:
2319 case Intrinsic::s390_vfeezfs:
2324 case Intrinsic::s390_vfenebs:
2325 case Intrinsic::s390_vfenehs:
2326 case Intrinsic::s390_vfenefs:
2331 case Intrinsic::s390_vfenezbs:
2332 case Intrinsic::s390_vfenezhs:
2333 case Intrinsic::s390_vfenezfs:
2338 case Intrinsic::s390_vistrbs:
2339 case Intrinsic::s390_vistrhs:
2340 case Intrinsic::s390_vistrfs:
2345 case Intrinsic::s390_vstrcbs:
2346 case Intrinsic::s390_vstrchs:
2347 case Intrinsic::s390_vstrcfs:
2352 case Intrinsic::s390_vstrczbs:
2353 case Intrinsic::s390_vstrczhs:
2354 case Intrinsic::s390_vstrczfs:
2359 case Intrinsic::s390_vstrsb:
2360 case Intrinsic::s390_vstrsh:
2361 case Intrinsic::s390_vstrsf:
2366 case Intrinsic::s390_vstrszb:
2367 case Intrinsic::s390_vstrszh:
2368 case Intrinsic::s390_vstrszf:
2373 case Intrinsic::s390_vfcedbs:
2374 case Intrinsic::s390_vfcesbs:
2379 case Intrinsic::s390_vfchdbs:
2380 case Intrinsic::s390_vfchsbs:
2385 case Intrinsic::s390_vfchedbs:
2386 case Intrinsic::s390_vfchesbs:
2391 case Intrinsic::s390_vftcidb:
2392 case Intrinsic::s390_vftcisb:
2397 case Intrinsic::s390_tdc:
2415 for (
unsigned I = 2;
I < NumOps; ++
I)
2418 assert(
Op->getNumValues() == 2 &&
"Expected only CC result and chain");
2424 return Intr.getNode();
2434 for (
unsigned I = 1;
I < NumOps; ++
I)
2438 return Intr.getNode();
2448 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
2449 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
2450 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
2475 auto *ConstOp1 = dyn_cast<ConstantSDNode>(
C.Op1.getNode());
2476 if (!ConstOp1 || ConstOp1->getValueSizeInBits(0) > 64)
2479 int64_t
Value = ConstOp1->getSExtValue();
2495 if (!
C.Op0.hasOneUse() ||
2501 auto *Load = cast<LoadSDNode>(
C.Op0);
2502 unsigned NumBits = Load->getMemoryVT().getSizeInBits();
2503 if ((NumBits != 8 && NumBits != 16) ||
2504 NumBits != Load->getMemoryVT().getStoreSizeInBits())
2509 auto *ConstOp1 = cast<ConstantSDNode>(
C.Op1);
2510 if (!ConstOp1 || ConstOp1->getValueSizeInBits(0) > 64)
2513 uint64_t Mask = (1 << NumBits) - 1;
2516 int64_t SignedValue = ConstOp1->getSExtValue();
2523 }
else if (NumBits == 8) {
2549 if (
C.Op0.getValueType() != MVT::i32 ||
2550 Load->getExtensionType() != ExtType) {
2552 Load->getBasePtr(), Load->getPointerInfo(),
2553 Load->getMemoryVT(), Load->getAlign(),
2554 Load->getMemOperand()->getFlags());
2560 if (
C.Op1.getValueType() != MVT::i32 ||
2561 Value != ConstOp1->getZExtValue())
2568 auto *Load = dyn_cast<LoadSDNode>(
Op.getNode());
2571 if (Load->getMemoryVT() == MVT::i8)
2574 switch (Load->getExtensionType()) {
2591 if (
C.Op0.getValueType() == MVT::i128)
2593 if (
C.Op0.getValueType() == MVT::f128)
2599 if (isa<ConstantFPSDNode>(
C.Op1))
2604 auto *ConstOp1 = dyn_cast<ConstantSDNode>(
C.Op1);
2605 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
2623 isUInt<16>(ConstOp1->getZExtValue()))
2628 isInt<16>(ConstOp1->getSExtValue()))
2634 unsigned Opcode0 =
C.Op0.getOpcode();
2641 C.Op0.getConstantOperandVal(1) == 0xffffffff)
2656 ((
N->getOperand(0) ==
C.Op0 &&
N->getOperand(1) ==
C.Op1) ||
2657 (
N->getOperand(0) ==
C.Op1 &&
N->getOperand(1) ==
C.Op0))) {
2661 Flags.setNoSignedWrap(
false);
2662 Flags.setNoUnsignedWrap(
false);
2681 auto *C1 = dyn_cast<ConstantFPSDNode>(
C.Op1);
2682 if (C1 && C1->isZero()) {
2701 if (
C.Op0.getOpcode() ==
ISD::SHL &&
C.Op0.getValueType() == MVT::i64 &&
2703 auto *C1 = dyn_cast<ConstantSDNode>(
C.Op0.getOperand(1));
2704 if (C1 && C1->getZExtValue() == 32) {
2705 SDValue ShlOp0 =
C.Op0.getOperand(0);
2709 cast<VTSDNode>(
N->getOperand(1))->getVT() == MVT::i32) {
2724 C.Op0.getOperand(0).getOpcode() ==
ISD::LOAD &&
2726 cast<ConstantSDNode>(
C.Op1)->getValueSizeInBits(0) <= 64 &&
2727 C.Op1->getAsZExtVal() == 0) {
2728 auto *L = cast<LoadSDNode>(
C.Op0.getOperand(0));
2729 if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <=
2730 C.Op0.getValueSizeInBits().getFixedValue()) {
2731 unsigned Type = L->getExtensionType();
2734 C.Op0 =
C.Op0.getOperand(0);
2744 auto *Shift = dyn_cast<ConstantSDNode>(
N.getOperand(1));
2748 uint64_t Amount = Shift->getZExtValue();
2749 if (Amount >=
N.getValueSizeInBits())
2764 unsigned ICmpType) {
2765 assert(Mask != 0 &&
"ANDs with zero should have been removed by now");
2787 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <=
Low) {
2793 if (EffectivelyUnsigned && CmpVal <
Low) {
2801 if (CmpVal == Mask) {
2807 if (EffectivelyUnsigned && CmpVal >= Mask -
Low && CmpVal < Mask) {
2813 if (EffectivelyUnsigned && CmpVal > Mask -
Low && CmpVal <= Mask) {
2821 if (EffectivelyUnsigned && CmpVal >= Mask -
High && CmpVal <
High) {
2827 if (EffectivelyUnsigned && CmpVal > Mask -
High && CmpVal <=
High) {
2856 if (
C.Op0.getValueType() == MVT::i128) {
2861 auto *Mask = dyn_cast<ConstantSDNode>(
C.Op1);
2862 if (Mask && Mask->getAPIntValue() == 0) {
2877 auto *ConstOp1 = dyn_cast<ConstantSDNode>(
C.Op1);
2880 uint64_t CmpVal = ConstOp1->getZExtValue();
2887 NewC.Op0 =
C.Op0.getOperand(0);
2888 NewC.Op1 =
C.Op0.getOperand(1);
2889 Mask = dyn_cast<ConstantSDNode>(NewC.Op1);
2892 MaskVal = Mask->getZExtValue();
2897 if (NewC.Op0.getValueType() != MVT::i64 ||
2912 MaskVal = -(CmpVal & -CmpVal);
2920 unsigned BitSize = NewC.Op0.getValueSizeInBits();
2921 unsigned NewCCMask, ShiftVal;
2923 NewC.Op0.getOpcode() ==
ISD::SHL &&
2925 (MaskVal >> ShiftVal != 0) &&
2926 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
2928 MaskVal >> ShiftVal,
2931 NewC.Op0 = NewC.Op0.getOperand(0);
2932 MaskVal >>= ShiftVal;
2934 NewC.Op0.getOpcode() ==
ISD::SRL &&
2936 (MaskVal << ShiftVal != 0) &&
2937 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
2939 MaskVal << ShiftVal,
2942 NewC.Op0 = NewC.Op0.getOperand(0);
2943 MaskVal <<= ShiftVal;
2954 if (Mask && Mask->getZExtValue() == MaskVal)
2959 C.CCMask = NewCCMask;
2967 if (
C.Op0.getValueType() != MVT::i128)
2985 bool Swap =
false, Invert =
false;
3004 C.CCMask ^=
C.CCValid;
3014 auto *Mask = dyn_cast<ConstantSDNode>(
C.Op0.getOperand(1));
3015 if (!Mask || Mask->getValueSizeInBits(0) > 64)
3018 if ((~Known.
Zero).getZExtValue() & ~Mask->getZExtValue())
3021 C.Op0 =
C.Op0.getOperand(0);
3033 C.CCValid = CCValid;
3036 C.CCMask =
CC < 4 ? 1 << (3 -
CC) : 0;
3039 C.CCMask =
CC < 4 ? ~(1 << (3 -
CC)) : -1;
3043 C.CCMask =
CC < 4 ? ~0U << (4 -
CC) : -1;
3046 C.CCMask =
CC < 4 ? ~(~0U << (4 -
CC)) : 0;
3050 C.CCMask =
CC < 4 ? ~0U << (3 -
CC) : -1;
3053 C.CCMask =
CC < 4 ? ~(~0U << (3 -
CC)) : 0;
3056 C.CCMask &= CCValid;
3064 bool IsSignaling =
false) {
3067 unsigned Opcode, CCValid;
3079 Comparison
C(CmpOp0, CmpOp1, Chain);
3081 if (
C.Op0.getValueType().isFloatingPoint()) {
3085 else if (!IsSignaling)
3107 C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
3128 if (!
C.Op1.getNode()) {
3130 switch (
C.Op0.getOpcode()) {
3157 return DAG.
getNode(
C.Opcode,
DL, VTs,
C.Chain,
C.Op0,
C.Op1);
3159 return DAG.
getNode(
C.Opcode,
DL, MVT::i32,
C.Op0,
C.Op1);
3168 Op0 = DAG.
getNode(Extend,
DL, MVT::i64, Op0);
3169 Op1 = DAG.
getNode(Extend,
DL, MVT::i64, Op1);
3194 unsigned CCValid,
unsigned CCMask) {
3223 case CmpMode::Int:
return 0;
3243 case CmpMode::FP:
return 0;
3244 case CmpMode::StrictFP:
return 0;
3245 case CmpMode::SignalingFP:
return 0;
3277 int Mask[] = { Start, -1, Start + 1, -1 };
3297 !Subtarget.hasVectorEnhancements1()) {
3311 SDValue Ops[2] = { Res, NewChain };
3320 return DAG.
getNode(Opcode,
DL, VTs, Chain, CmpOp0, CmpOp1);
3322 return DAG.
getNode(Opcode,
DL, VT, CmpOp0, CmpOp1);
3335 bool IsSignaling)
const {
3338 assert (!IsSignaling || Chain);
3339 CmpMode Mode = IsSignaling ? CmpMode::SignalingFP :
3340 Chain ? CmpMode::StrictFP : IsFP ? CmpMode::FP : CmpMode::Int;
3341 bool Invert =
false;
3349 assert(IsFP &&
"Unexpected integer comparison");
3351 DL, VT, CmpOp1, CmpOp0, Chain);
3353 DL, VT, CmpOp0, CmpOp1, Chain);
3357 LT.getValue(1),
GE.getValue(1));
3366 assert(IsFP &&
"Unexpected integer comparison");
3368 DL, VT, CmpOp1, CmpOp0, Chain);
3370 DL, VT, CmpOp0, CmpOp1, Chain);
3374 LT.getValue(1),
GT.getValue(1));
3383 Cmp = getVectorCmp(DAG, Opcode,
DL, VT, CmpOp0, CmpOp1, Chain);
3387 Cmp = getVectorCmp(DAG, Opcode,
DL, VT, CmpOp1, CmpOp0, Chain);
3392 Chain =
Cmp.getValue(1);
3400 if (Chain && Chain.
getNode() !=
Cmp.getNode()) {
3413 EVT VT =
Op.getValueType();
3415 return lowerVectorSETCC(DAG,
DL, VT,
CC, CmpOp0, CmpOp1);
3417 Comparison
C(
getCmp(DAG, CmpOp0, CmpOp1,
CC,
DL));
3424 bool IsSignaling)
const {
3430 EVT VT =
Op.getNode()->getValueType(0);
3432 SDValue Res = lowerVectorSETCC(DAG,
DL, VT,
CC, CmpOp0, CmpOp1,
3433 Chain, IsSignaling);
3437 Comparison
C(
getCmp(DAG, CmpOp0, CmpOp1,
CC,
DL, Chain, IsSignaling));
3452 Comparison
C(
getCmp(DAG, CmpOp0, CmpOp1,
CC,
DL));
3489 Comparison
C(
getCmp(DAG, CmpOp0, CmpOp1,
CC,
DL));
3497 cast<ConstantSDNode>(
C.Op1)->getValueSizeInBits(0) <= 64 &&
3498 C.Op1->getAsZExtVal() == 0) {
3506 SDValue Ops[] = {TrueOp, FalseOp,
3580 Chain = DAG.
getCopyToReg(Chain,
DL, SystemZ::R2D, GOTOffset, Glue);
3587 Node->getValueType(0),
3599 assert(Mask &&
"Missing call preserved mask for calling convention");
3607 Chain = DAG.
getNode(Opcode,
DL, NodeTys, Ops);
3614SDValue SystemZTargetLowering::lowerThreadPointer(
const SDLoc &
DL,
3646 SDValue TP = lowerThreadPointer(
DL, DAG);
3754 if (
CP->isMachineConstantPoolEntry())
3773 unsigned Depth =
Op.getConstantOperandVal(0);
3780 int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
3809 unsigned Depth =
Op.getConstantOperandVal(0);
3817 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
3819 int Offset = TFL->getReturnAddressOffset(MF);
3830 &SystemZ::GR64BitRegClass);
3838 EVT InVT =
In.getValueType();
3839 EVT ResVT =
Op.getValueType();
3844 if (
auto *LoadN = dyn_cast<LoadSDNode>(In))
3847 LoadN->getBasePtr(), LoadN->getMemOperand());
3853 if (InVT == MVT::i32 && ResVT == MVT::f32) {
3855 if (Subtarget.hasHighWord()) {
3859 MVT::i64,
SDValue(U64, 0), In);
3867 DL, MVT::f32, Out64);
3869 if (InVT == MVT::f32 && ResVT == MVT::i32) {
3872 MVT::f64,
SDValue(U64, 0), In);
3874 if (Subtarget.hasHighWord())
3888 return lowerVASTART_XPLINK(
Op, DAG);
3890 return lowerVASTART_ELF(
Op, DAG);
3905 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
3919 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
3923 const unsigned NumFields = 4;
3934 for (
unsigned I = 0;
I < NumFields; ++
I) {
3939 MemOps[
I] = DAG.
getStore(Chain,
DL, Fields[
I], FieldAddr,
3951 const Value *DstSV = cast<SrcValueSDNode>(
Op.getOperand(3))->getValue();
3952 const Value *SrcSV = cast<SrcValueSDNode>(
Op.getOperand(4))->getValue();
3958 Align(8),
false,
false,
3964SystemZTargetLowering::lowerDYNAMIC_STACKALLOC(
SDValue Op,
3967 return lowerDYNAMIC_STACKALLOC_XPLINK(
Op, DAG);
3969 return lowerDYNAMIC_STACKALLOC_ELF(
Op, DAG);
3973SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(
SDValue Op,
3985 uint64_t AlignVal = (RealignOpt ?
Align->getAsZExtVal() : 0);
3988 uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
3989 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
3995 if (ExtraAlignSpace)
3999 bool IsSigned =
false;
4000 bool DoesNotReturn =
false;
4001 bool IsReturnValueUsed =
false;
4002 EVT VT =
Op.getValueType();
4013 Register SPReg = Regs.getStackPointerRegister();
4024 if (ExtraAlignSpace) {
4036SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_ELF(
SDValue Op,
4050 uint64_t AlignVal = (RealignOpt ?
Align->getAsZExtVal() : 0);
4053 uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
4054 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4065 Backchain = DAG.
getLoad(MVT::i64,
DL, Chain, getBackchainAddress(OldSP, DAG),
4069 if (ExtraAlignSpace)
4077 DAG.
getVTList(MVT::i64, MVT::Other), Chain, OldSP, NeededSpace);
4093 if (RequiredAlign > StackAlign) {
4103 Chain = DAG.
getStore(Chain,
DL, Backchain, getBackchainAddress(NewSP, DAG),
4110SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
4119 EVT VT =
Op.getValueType();
4126 Op.getOperand(1), Ops[1], Ops[0]);
4127 else if (Subtarget.hasMiscellaneousExtensions2())
4132 Op.getOperand(0),
Op.getOperand(1), Ops[1], Ops[0]);
4156 LL, RL, Ops[1], Ops[0]);
4167 EVT VT =
Op.getValueType();
4174 Op.getOperand(1), Ops[1], Ops[0]);
4180 Op.getOperand(0),
Op.getOperand(1), Ops[1], Ops[0]);
4188 EVT VT =
Op.getValueType();
4208 EVT VT =
Op.getValueType();
4215 Op.getOperand(0),
Op.getOperand(1), Ops[1], Ops[0]);
4220 assert(
Op.getValueType() == MVT::i64 &&
"Should be 64-bit operation");
4223 SDValue Ops[] = {
Op.getOperand(0),
Op.getOperand(1)};
4232 if ((Masks[0] >> 32) == 0xffffffff &&
uint32_t(Masks[1]) == 0xffffffff)
4234 else if ((Masks[1] >> 32) == 0xffffffff &&
uint32_t(Masks[0]) == 0xffffffff)
4250 if (!isInt<16>(
Value))
4271 MVT::i64, HighOp, Low32);
4282 if (
N->getValueType(0) == MVT::i128) {
4283 unsigned BaseOp = 0;
4284 unsigned FlagOp = 0;
4285 bool IsBorrow =
false;
4286 switch (
Op.getOpcode()) {
4309 unsigned BaseOp = 0;
4310 unsigned CCValid = 0;
4311 unsigned CCMask = 0;
4313 switch (
Op.getOpcode()) {
4341 if (
N->getValueType(1) == MVT::i1)
4364 MVT VT =
N->getSimpleValueType(0);
4375 if (VT == MVT::i128) {
4376 unsigned BaseOp = 0;
4377 unsigned FlagOp = 0;
4378 bool IsBorrow =
false;
4379 switch (
Op.getOpcode()) {
4406 unsigned BaseOp = 0;
4407 unsigned CCValid = 0;
4408 unsigned CCMask = 0;
4410 switch (
Op.getOpcode()) {
4439 if (
N->getValueType(1) == MVT::i1)
4447 EVT VT =
Op.getValueType();
4449 Op =
Op.getOperand(0);
4497 if (NumSignificantBits == 0)
4503 BitSize = std::min(BitSize, OrigBitSize);
4512 for (int64_t
I = BitSize / 2;
I >= 8;
I =
I / 2) {
4514 if (BitSize != OrigBitSize)
4551 auto *
Node = cast<AtomicSDNode>(
Op.getNode());
4553 (
Node->getMemoryVT() == MVT::i128 ||
Node->getMemoryVT() == MVT::f128) &&
4554 "Only custom lowering i128 or f128.");
4566 EVT PtrVT =
Addr.getValueType();
4567 EVT WideVT = MVT::i32;
4590 unsigned Opcode)
const {
4591 auto *
Node = cast<AtomicSDNode>(
Op.getNode());
4594 EVT NarrowVT =
Node->getMemoryVT();
4595 EVT WideVT = MVT::i32;
4596 if (NarrowVT == WideVT)
4608 if (
auto *Const = dyn_cast<ConstantSDNode>(Src2)) {
4613 SDValue AlignedAddr, BitShift, NegBitShift;
4631 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
4650 auto *
Node = cast<AtomicSDNode>(
Op.getNode());
4651 EVT MemVT =
Node->getMemoryVT();
4652 if (MemVT == MVT::i32 || MemVT == MVT::i64) {
4654 assert(
Op.getValueType() == MemVT &&
"Mismatched VTs");
4655 assert(Subtarget.hasInterlockedAccess1() &&
4656 "Should have been expanded by AtomicExpand pass.");
4662 Node->getChain(),
Node->getBasePtr(), NegSrc2,
4663 Node->getMemOperand());
4672 auto *
Node = cast<AtomicSDNode>(
Op.getNode());
4680 if (
Node->getMemoryVT() == MVT::i128) {
4689 EVT NarrowVT =
Node->getMemoryVT();
4690 EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
4691 if (NarrowVT == WideVT) {
4693 SDValue Ops[] = { ChainIn,
Addr, CmpVal, SwapVal };
4695 DL, Tys, Ops, NarrowVT, MMO);
4709 SDValue AlignedAddr, BitShift, NegBitShift;
4714 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
4717 VTList, Ops, NarrowVT, MMO);
4731SystemZTargetLowering::getTargetMMOFlags(
const Instruction &
I)
const {
4736 if (
auto *SI = dyn_cast<StoreInst>(&
I))
4739 if (
auto *LI = dyn_cast<LoadInst>(&
I))
4742 if (
auto *AI = dyn_cast<AtomicRMWInst>(&
I))
4745 if (
auto *AI = dyn_cast<AtomicCmpXchgInst>(&
I))
4757 "in GHC calling convention");
4759 Regs->getStackPointerRegister(),
Op.getValueType());
4770 "in GHC calling convention");
4777 if (StoreBackchain) {
4779 Chain,
DL, Regs->getStackPointerRegister(), MVT::i64);
4780 Backchain = DAG.
getLoad(MVT::i64,
DL, Chain, getBackchainAddress(OldSP, DAG),
4784 Chain = DAG.
getCopyToReg(Chain,
DL, Regs->getStackPointerRegister(), NewSP);
4787 Chain = DAG.
getStore(Chain,
DL, Backchain, getBackchainAddress(NewSP, DAG),
4795 bool IsData =
Op.getConstantOperandVal(4);
4798 return Op.getOperand(0);
4801 bool IsWrite =
Op.getConstantOperandVal(2);
4803 auto *
Node = cast<MemIntrinsicSDNode>(
Op.getNode());
4807 Node->getVTList(), Ops,
4808 Node->getMemoryVT(),
Node->getMemOperand());
4820SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(
SDValue Op,
4822 unsigned Opcode, CCValid;
4824 assert(
Op->getNumValues() == 2 &&
"Expected only CC result and chain");
4835SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(
SDValue Op,
4837 unsigned Opcode, CCValid;
4840 if (
Op->getNumValues() == 1)
4842 assert(
Op->getNumValues() == 2 &&
"Expected a CC and non-CC result");
4847 unsigned Id =
Op.getConstantOperandVal(0);
4849 case Intrinsic::thread_pointer:
4850 return lowerThreadPointer(
SDLoc(
Op), DAG);
4852 case Intrinsic::s390_vpdi:
4854 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4856 case Intrinsic::s390_vperm:
4858 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4860 case Intrinsic::s390_vuphb:
4861 case Intrinsic::s390_vuphh:
4862 case Intrinsic::s390_vuphf:
4866 case Intrinsic::s390_vuplhb:
4867 case Intrinsic::s390_vuplhh:
4868 case Intrinsic::s390_vuplhf:
4872 case Intrinsic::s390_vuplb:
4873 case Intrinsic::s390_vuplhw:
4874 case Intrinsic::s390_vuplf:
4878 case Intrinsic::s390_vupllb:
4879 case Intrinsic::s390_vupllh:
4880 case Intrinsic::s390_vupllf:
4884 case Intrinsic::s390_vsumb:
4885 case Intrinsic::s390_vsumh:
4886 case Intrinsic::s390_vsumgh:
4887 case Intrinsic::s390_vsumgf:
4888 case Intrinsic::s390_vsumqf:
4889 case Intrinsic::s390_vsumqg:
4891 Op.getOperand(1),
Op.getOperand(2));
4893 case Intrinsic::s390_vaq:
4895 Op.getOperand(1),
Op.getOperand(2));
4896 case Intrinsic::s390_vaccb:
4897 case Intrinsic::s390_vacch:
4898 case Intrinsic::s390_vaccf:
4899 case Intrinsic::s390_vaccg:
4900 case Intrinsic::s390_vaccq:
4902 Op.getOperand(1),
Op.getOperand(2));
4903 case Intrinsic::s390_vacq:
4905 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4906 case Intrinsic::s390_vacccq:
4908 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4910 case Intrinsic::s390_vsq:
4912 Op.getOperand(1),
Op.getOperand(2));
4913 case Intrinsic::s390_vscbib:
4914 case Intrinsic::s390_vscbih:
4915 case Intrinsic::s390_vscbif:
4916 case Intrinsic::s390_vscbig:
4917 case Intrinsic::s390_vscbiq:
4919 Op.getOperand(1),
Op.getOperand(2));
4920 case Intrinsic::s390_vsbiq:
4922 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4923 case Intrinsic::s390_vsbcbiq:
4925 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4946 { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
4949 { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
4952 { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
4955 { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
4958 { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
4961 { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
4964 { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
4967 { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
4970 { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
4973 { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
4976 { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
4979 { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
4982 { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
4996 OpNo0 = OpNo1 = OpNos[1];
4997 }
else if (OpNos[1] < 0) {
4998 OpNo0 = OpNo1 = OpNos[0];
5016 unsigned &OpNo0,
unsigned &OpNo1) {
5017 int OpNos[] = { -1, -1 };
5030 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5032 OpNos[ModelOpNo] = RealOpNo;
5040 unsigned &OpNo0,
unsigned &OpNo1) {
5057 int Elt = Bytes[
From];
5060 Transform[
From] = -1;
5062 while (
P.Bytes[To] != Elt) {
5067 Transform[
From] = To;
5090 if (
auto *VSN = dyn_cast<ShuffleVectorSDNode>(ShuffleOp)) {
5091 Bytes.
resize(NumElements * BytesPerElement, -1);
5092 for (
unsigned I = 0;
I < NumElements; ++
I) {
5093 int Index = VSN->getMaskElt(
I);
5095 for (
unsigned J = 0; J < BytesPerElement; ++J)
5096 Bytes[
I * BytesPerElement + J] =
Index * BytesPerElement + J;
5101 isa<ConstantSDNode>(ShuffleOp.
getOperand(1))) {
5103 Bytes.
resize(NumElements * BytesPerElement, -1);
5104 for (
unsigned I = 0;
I < NumElements; ++
I)
5105 for (
unsigned J = 0; J < BytesPerElement; ++J)
5106 Bytes[
I * BytesPerElement + J] =
Index * BytesPerElement + J;
5117 unsigned BytesPerElement,
int &
Base) {
5119 for (
unsigned I = 0;
I < BytesPerElement; ++
I) {
5120 if (Bytes[Start +
I] >= 0) {
5121 unsigned Elem = Bytes[Start +
I];
5125 if (
unsigned(
Base) % Bytes.
size() + BytesPerElement > Bytes.
size())
5127 }
else if (
unsigned(
Base) != Elem -
I)
5140 unsigned &StartIndex,
unsigned &OpNo0,
5142 int OpNos[] = { -1, -1 };
5144 for (
unsigned I = 0;
I < 16; ++
I) {
5151 Shift = ExpectedShift;
5152 else if (Shift != ExpectedShift)
5156 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5158 OpNos[ModelOpNo] = RealOpNo;
5195 N =
N->getOperand(0);
5197 if (
auto *
Op = dyn_cast<ConstantSDNode>(
N->getOperand(0)))
5198 return Op->getZExtValue() == 0;
5204 for (
unsigned I = 0;
I < Num ;
I++)
5216 for (
unsigned I = 0;
I < 2; ++
I)
5220 unsigned StartIndex, OpNo0, OpNo1;
5229 if (ZeroVecIdx != UINT32_MAX) {
5230 bool MaskFirst =
true;
5235 if (OpNo == ZeroVecIdx &&
I == 0) {
5240 if (OpNo != ZeroVecIdx && Byte == 0) {
5247 if (ZeroIdx != -1) {
5250 if (Bytes[
I] >= 0) {
5253 if (OpNo == ZeroVecIdx)
5263 SDValue Src = ZeroVecIdx == 0 ? Ops[1] : Ops[0];
5281 (!Ops[1].
isUndef() ? Ops[1] : Ops[0]), Op2);
5286struct GeneralShuffle {
5287 GeneralShuffle(
EVT vt) : VT(vt), UnpackFromEltSize(UINT_MAX) {}
5291 void tryPrepareForUnpack();
5292 bool unpackWasPrepared() {
return UnpackFromEltSize <= 4; }
5307 unsigned UnpackFromEltSize;
5312void GeneralShuffle::addUndef() {
5314 for (
unsigned I = 0;
I < BytesPerElement; ++
I)
5315 Bytes.push_back(-1);
5324bool GeneralShuffle::add(
SDValue Op,
unsigned Elem) {
5330 EVT FromVT =
Op.getNode() ?
Op.getValueType() : VT;
5335 if (FromBytesPerElement < BytesPerElement)
5339 (FromBytesPerElement - BytesPerElement));
5342 while (
Op.getNode()) {
5344 Op =
Op.getOperand(0);
5360 }
else if (
Op.isUndef()) {
5369 for (; OpNo < Ops.size(); ++OpNo)
5370 if (Ops[OpNo] ==
Op)
5372 if (OpNo == Ops.size())
5377 for (
unsigned I = 0;
I < BytesPerElement; ++
I)
5378 Bytes.push_back(
Base +
I);
5387 if (Ops.size() == 0)
5391 tryPrepareForUnpack();
5394 if (Ops.size() == 1)
5395 Ops.push_back(DAG.
getUNDEF(MVT::v16i8));
5406 unsigned Stride = 1;
5407 for (; Stride * 2 < Ops.size(); Stride *= 2) {
5408 for (
unsigned I = 0;
I < Ops.size() - Stride;
I += Stride * 2) {
5409 SDValue SubOps[] = { Ops[
I], Ops[
I + Stride] };
5418 else if (OpNo ==
I + Stride)
5429 if (NewBytes[J] >= 0) {
5431 "Invalid double permute");
5434 assert(NewBytesMap[J] < 0 &&
"Invalid double permute");
5440 if (NewBytes[J] >= 0)
5448 Ops[1] = Ops[Stride];
5456 unsigned OpNo0, OpNo1;
5458 if (unpackWasPrepared() && Ops[1].
isUndef())
5460 else if (
const Permute *
P =
matchPermute(Bytes, OpNo0, OpNo1))
5465 Op = insertUnpackIfPrepared(DAG,
DL,
Op);
5472 dbgs() << Msg.c_str() <<
" { ";
5473 for (
unsigned i = 0; i < Bytes.
size(); i++)
5474 dbgs() << Bytes[i] <<
" ";
5482void GeneralShuffle::tryPrepareForUnpack() {
5484 if (ZeroVecOpNo == UINT32_MAX || Ops.size() == 1)
5489 if (Ops.size() > 2 &&
5494 UnpackFromEltSize = 1;
5495 for (; UnpackFromEltSize <= 4; UnpackFromEltSize *= 2) {
5496 bool MatchUnpack =
true;
5499 unsigned ToEltSize = UnpackFromEltSize * 2;
5500 bool IsZextByte = (Elt % ToEltSize) < UnpackFromEltSize;
5503 if (Bytes[Elt] != -1) {
5505 if (IsZextByte != (OpNo == ZeroVecOpNo)) {
5506 MatchUnpack =
false;
5512 if (Ops.size() == 2) {
5515 if (SrcBytes[i] != -1 && SrcBytes[i] % 16 !=
int(i)) {
5516 UnpackFromEltSize = UINT_MAX;
5523 if (UnpackFromEltSize > 4)
5526 LLVM_DEBUG(
dbgs() <<
"Preparing for final unpack of element size "
5527 << UnpackFromEltSize <<
". Zero vector is Op#" << ZeroVecOpNo
5529 dumpBytes(Bytes,
"Original Bytes vector:"););
5534 Elt += UnpackFromEltSize;
5535 for (
unsigned i = 0; i < UnpackFromEltSize; i++, Elt++,
B++)
5536 Bytes[
B] = Bytes[Elt];
5542 Ops.erase(&Ops[ZeroVecOpNo]);
5544 if (Bytes[
I] >= 0) {
5546 if (OpNo > ZeroVecOpNo)
5557 if (!unpackWasPrepared())
5559 unsigned InBits = UnpackFromEltSize * 8;
5563 unsigned OutBits = InBits * 2;
5572 if (!
Op.getOperand(
I).isUndef())
5588 if (
Value.isUndef())
5641 GeneralShuffle GS(VT);
5643 bool FoundOne =
false;
5644 for (
unsigned I = 0;
I < NumElements; ++
I) {
5647 Op =
Op.getOperand(0);
5650 unsigned Elem =
Op.getConstantOperandVal(1);
5651 if (!GS.add(
Op.getOperand(0), Elem))
5654 }
else if (
Op.isUndef()) {
5668 if (!ResidueOps.
empty()) {
5669 while (ResidueOps.
size() < NumElements)
5671 for (
auto &
Op : GS.Ops) {
5672 if (!
Op.getNode()) {
5678 return GS.getNode(DAG,
SDLoc(BVN));
5681bool SystemZTargetLowering::isVectorElementLoad(
SDValue Op)
const {
5682 if (
Op.getOpcode() ==
ISD::LOAD && cast<LoadSDNode>(
Op)->isUnindexed())
5684 if (
auto *AL = dyn_cast<AtomicSDNode>(
Op))
5698 unsigned int NumElements = Elems.
size();
5699 unsigned int Count = 0;
5700 for (
auto Elem : Elems) {
5701 if (!Elem.isUndef()) {
5704 else if (Elem != Single) {
5724 if (
Single.getNode() && (Count > 1 || isVectorElementLoad(Single)))
5728 bool AllLoads =
true;
5729 for (
auto Elem : Elems)
5730 if (!isVectorElementLoad(Elem)) {
5736 if (VT == MVT::v2i64 && !AllLoads)
5740 if (VT == MVT::v2f64 && !AllLoads)
5750 if (VT == MVT::v4f32 && !AllLoads) {
5764 DL, MVT::v2i64, Op01, Op23);
5772 unsigned NumConstants = 0;
5773 for (
unsigned I = 0;
I < NumElements; ++
I) {
5787 if (NumConstants > 0) {
5788 for (
unsigned I = 0;
I < NumElements; ++
I)
5799 std::map<const SDNode*, unsigned> UseCounts;
5800 SDNode *LoadMaxUses =
nullptr;
5801 for (
unsigned I = 0;
I < NumElements; ++
I)
5802 if (isVectorElementLoad(Elems[
I])) {
5803 SDNode *Ld = Elems[
I].getNode();
5805 if (LoadMaxUses ==
nullptr || UseCounts[LoadMaxUses] < UseCounts[Ld])
5808 if (LoadMaxUses !=
nullptr) {
5809 ReplicatedVal =
SDValue(LoadMaxUses, 0);
5813 unsigned I1 = NumElements / 2 - 1;
5814 unsigned I2 = NumElements - 1;
5815 bool Def1 = !Elems[
I1].isUndef();
5816 bool Def2 = !Elems[I2].isUndef();
5830 for (
unsigned I = 0;
I < NumElements; ++
I)
5831 if (!
Done[
I] && !Elems[
I].
isUndef() && Elems[
I] != ReplicatedVal)
5839 auto *BVN = cast<BuildVectorSDNode>(
Op.getNode());
5841 EVT VT =
Op.getValueType();
5843 if (BVN->isConstant()) {
5862 for (
unsigned I = 0;
I < NumElements; ++
I)
5863 Ops[
I] =
Op.getOperand(
I);
5864 return buildVector(DAG,
DL, VT, Ops);
5869 auto *VSN = cast<ShuffleVectorSDNode>(
Op.getNode());
5871 EVT VT =
Op.getValueType();
5874 if (VSN->isSplat()) {
5876 unsigned Index = VSN->getSplatIndex();
5878 "Splat index should be defined and in first operand");
5888 GeneralShuffle
GS(VT);
5889 for (
unsigned I = 0;
I < NumElements; ++
I) {
5890 int Elt = VSN->getMaskElt(
I);
5893 else if (!
GS.add(
Op.getOperand(
unsigned(Elt) / NumElements),
5894 unsigned(Elt) % NumElements))
5897 return GS.getNode(DAG,
SDLoc(VSN));
5916 EVT VT =
Op.getValueType();
5921 if (VT == MVT::v2f64 &&
5941SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(
SDValue Op,
5947 EVT VT =
Op.getValueType();
5951 if (
auto *CIndexN = dyn_cast<ConstantSDNode>(Op1)) {
5966SDValue SystemZTargetLowering::
5969 EVT OutVT =
Op.getValueType();
5979 }
while (FromBits != ToBits);
5984SDValue SystemZTargetLowering::
5988 EVT OutVT =
Op.getValueType();
5992 unsigned NumInPerOut = InNumElts / OutNumElts;
5998 unsigned ZeroVecElt = InNumElts;
5999 for (
unsigned PackedElt = 0; PackedElt < OutNumElts; PackedElt++) {
6000 unsigned MaskElt = PackedElt * NumInPerOut;
6001 unsigned End = MaskElt + NumInPerOut - 1;
6002 for (; MaskElt <
End; MaskElt++)
6003 Mask[MaskElt] = ZeroVecElt++;
6004 Mask[MaskElt] = PackedElt;
6011 unsigned ByScalar)
const {
6016 EVT VT =
Op.getValueType();
6020 if (
auto *BVN = dyn_cast<BuildVectorSDNode>(Op1)) {
6021 APInt SplatBits, SplatUndef;
6022 unsigned SplatBitSize;
6026 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6027 ElemBitSize,
true) &&
6028 SplatBitSize == ElemBitSize) {
6031 return DAG.
getNode(ByScalar,
DL, VT, Op0, Shift);
6040 return DAG.
getNode(ByScalar,
DL, VT, Op0, Shift);
6046 if (
auto *VSN = dyn_cast<ShuffleVectorSDNode>(Op1)) {
6047 if (VSN->isSplat()) {
6049 unsigned Index = VSN->getSplatIndex();
6051 "Splat index should be defined and in first operand");
6058 return DAG.
getNode(ByScalar,
DL, VT, Op0, Shift);
6070 MVT ResultVT =
Op.getSimpleValueType();
6072 unsigned Check =
Op.getConstantOperandVal(1);
6074 unsigned TDCMask = 0;
6108 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
6119 return DAG.
getLoad(MVT::i64,
DL, Chain, StackPtr, MPI);
6124 switch (
Op.getOpcode()) {
6126 return lowerFRAMEADDR(
Op, DAG);
6128 return lowerRETURNADDR(
Op, DAG);
6130 return lowerBR_CC(
Op, DAG);
6132 return lowerSELECT_CC(
Op, DAG);
6134 return lowerSETCC(
Op, DAG);
6136 return lowerSTRICT_FSETCC(
Op, DAG,
false);
6138 return lowerSTRICT_FSETCC(
Op, DAG,
true);
6140 return lowerGlobalAddress(cast<GlobalAddressSDNode>(
Op), DAG);
6142 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(
Op), DAG);
6144 return lowerBlockAddress(cast<BlockAddressSDNode>(
Op), DAG);
6146 return lowerJumpTable(cast<JumpTableSDNode>(
Op), DAG);
6148 return lowerConstantPool(cast<ConstantPoolSDNode>(
Op), DAG);
6150 return lowerBITCAST(
Op, DAG);
6152 return lowerVASTART(
Op, DAG);
6154 return lowerVACOPY(
Op, DAG);
6156 return lowerDYNAMIC_STACKALLOC(
Op, DAG);
6158 return lowerGET_DYNAMIC_AREA_OFFSET(
Op, DAG);
6160 return lowerSMUL_LOHI(
Op, DAG);
6162 return lowerUMUL_LOHI(
Op, DAG);
6164 return lowerSDIVREM(
Op, DAG);
6166 return lowerUDIVREM(
Op, DAG);
6171 return lowerXALUO(
Op, DAG);
6174 return lowerUADDSUBO_CARRY(
Op, DAG);
6176 return lowerOR(
Op, DAG);
6178 return lowerCTPOP(
Op, DAG);
6180 return lowerVECREDUCE_ADD(
Op, DAG);
6182 return lowerATOMIC_FENCE(
Op, DAG);
6187 return lowerATOMIC_LDST_I128(
Op, DAG);
6191 return lowerATOMIC_LOAD_SUB(
Op, DAG);
6209 return lowerATOMIC_CMP_SWAP(
Op, DAG);
6211 return lowerSTACKSAVE(
Op, DAG);
6213 return lowerSTACKRESTORE(
Op, DAG);
6215 return lowerPREFETCH(
Op, DAG);
6217 return lowerINTRINSIC_W_CHAIN(
Op, DAG);
6219 return lowerINTRINSIC_WO_CHAIN(
Op, DAG);
6221 return lowerBUILD_VECTOR(
Op, DAG);
6223 return lowerVECTOR_SHUFFLE(
Op, DAG);
6225 return lowerSCALAR_TO_VECTOR(
Op, DAG);
6227 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6229 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6231 return lowerSIGN_EXTEND_VECTOR_INREG(
Op, DAG);
6233 return lowerZERO_EXTEND_VECTOR_INREG(
Op, DAG);
6243 return lowerIS_FPCLASS(
Op, DAG);
6245 return lowerGET_ROUNDING(
Op, DAG);
6247 return lowerREADCYCLECOUNTER(
Op, DAG);
6261 &SystemZ::FP128BitRegClass);
6270 SystemZ::REG_SEQUENCE, SL, MVT::f128,
6285 &SystemZ::FP128BitRegClass);
6303 switch (
N->getOpcode()) {
6307 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1) };
6310 DL, Tys, Ops, MVT::i128, MMO);
6313 if (
N->getValueType(0) == MVT::f128)
6327 SDValue Ops[] = {
N->getOperand(0), Val,
N->getOperand(2)};
6330 DL, Tys, Ops, MVT::i128, MMO);
6333 if (cast<AtomicSDNode>(
N)->getSuccessOrdering() ==
6336 MVT::Other, Res), 0);
6343 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
6348 DL, Tys, Ops, MVT::i128, MMO);
6359 if (
N->getValueType(0) == MVT::i128 && Src.getValueType() == MVT::f128 &&
6379#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME
6490 OPCODE(ATOMIC_LOADW_ADD);
6491 OPCODE(ATOMIC_LOADW_SUB);
6492 OPCODE(ATOMIC_LOADW_AND);
6494 OPCODE(ATOMIC_LOADW_XOR);
6495 OPCODE(ATOMIC_LOADW_NAND);
6496 OPCODE(ATOMIC_LOADW_MIN);
6497 OPCODE(ATOMIC_LOADW_MAX);
6498 OPCODE(ATOMIC_LOADW_UMIN);
6499 OPCODE(ATOMIC_LOADW_UMAX);
6500 OPCODE(ATOMIC_CMP_SWAPW);
6503 OPCODE(ATOMIC_STORE_128);
6504 OPCODE(ATOMIC_CMP_SWAP_128);
6519bool SystemZTargetLowering::canTreatAsByteVector(
EVT VT)
const {
6520 if (!Subtarget.hasVector())
6534 DAGCombinerInfo &DCI,
6542 unsigned Opcode =
Op.getOpcode();
6545 Op =
Op.getOperand(0);
6547 canTreatAsByteVector(
Op.getValueType())) {
6556 BytesPerElement,
First))
6563 if (Byte % BytesPerElement != 0)
6566 Index = Byte / BytesPerElement;
6570 canTreatAsByteVector(
Op.getValueType())) {
6573 EVT OpVT =
Op.getValueType();
6575 if (OpBytesPerElement < BytesPerElement)
6579 unsigned End = (
Index + 1) * BytesPerElement;
6580 if (
End % OpBytesPerElement != 0)
6583 Op =
Op.getOperand(
End / OpBytesPerElement - 1);
6584 if (!
Op.getValueType().isInteger()) {
6587 DCI.AddToWorklist(
Op.getNode());
6592 DCI.AddToWorklist(
Op.getNode());
6599 canTreatAsByteVector(
Op.getValueType()) &&
6600 canTreatAsByteVector(
Op.getOperand(0).getValueType())) {
6602 EVT ExtVT =
Op.getValueType();
6603 EVT OpVT =
Op.getOperand(0).getValueType();
6606 unsigned Byte =
Index * BytesPerElement;
6607 unsigned SubByte =
Byte % ExtBytesPerElement;
6608 unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
6609 if (SubByte < MinSubByte ||
6610 SubByte + BytesPerElement > ExtBytesPerElement)
6613 Byte =
Byte / ExtBytesPerElement * OpBytesPerElement;
6615 Byte += SubByte - MinSubByte;
6616 if (Byte % BytesPerElement != 0)
6618 Op =
Op.getOperand(0);
6625 if (
Op.getValueType() != VecVT) {
6627 DCI.AddToWorklist(
Op.getNode());
6637SDValue SystemZTargetLowering::combineTruncateExtract(
6646 if (canTreatAsByteVector(VecVT)) {
6647 if (
auto *IndexN = dyn_cast<ConstantSDNode>(
Op.getOperand(1))) {
6650 if (BytesPerElement % TruncBytes == 0) {
6656 unsigned Scale = BytesPerElement / TruncBytes;
6657 unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
6663 EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
6664 return combineExtract(
DL, ResVT, VecVT, Vec, NewIndex, DCI,
true);
6672SDValue SystemZTargetLowering::combineZERO_EXTEND(
6673 SDNode *
N, DAGCombinerInfo &DCI)
const {
6677 EVT VT =
N->getValueType(0);
6679 auto *TrueOp = dyn_cast<ConstantSDNode>(N0.
getOperand(0));
6680 auto *FalseOp = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
6681 if (TrueOp && FalseOp) {
6691 DCI.CombineTo(N0.
getNode(), TruncSelect);
6721SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
6722 SDNode *
N, DAGCombinerInfo &DCI)
const {
6728 EVT VT =
N->getValueType(0);
6729 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
6742SDValue SystemZTargetLowering::combineSIGN_EXTEND(
6743 SDNode *
N, DAGCombinerInfo &DCI)
const {
6749 EVT VT =
N->getValueType(0);
6751 auto *SraAmt = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
6754 if (
auto *ShlAmt = dyn_cast<ConstantSDNode>(Inner.
getOperand(1))) {
6756 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;
6757 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;
6773SDValue SystemZTargetLowering::combineMERGE(
6774 SDNode *
N, DAGCombinerInfo &DCI)
const {
6776 unsigned Opcode =
N->getOpcode();
6784 if (Op1 ==
N->getOperand(0))
6789 if (ElemBytes <= 4) {
6797 DCI.AddToWorklist(Op1.
getNode());
6800 DCI.AddToWorklist(
Op.getNode());
6809 LoPart = HiPart =
nullptr;
6813 UI != UIEnd; ++UI) {
6815 if (UI.getUse().getResNo() != 0)
6820 bool IsLoPart =
true;
6845 LoPart = HiPart =
nullptr;
6849 UI != UIEnd; ++UI) {
6851 if (UI.getUse().getResNo() != 0)
6857 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
6860 switch (
User->getConstantOperandVal(1)) {
6861 case SystemZ::subreg_l64:
6866 case SystemZ::subreg_h64:
6878SDValue SystemZTargetLowering::combineLOAD(
6879 SDNode *
N, DAGCombinerInfo &DCI)
const {
6881 EVT LdVT =
N->getValueType(0);
6896 LD->getPointerInfo(),
LD->getOriginalAlign(),
6897 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
6899 DCI.CombineTo(HiPart, EltLoad,
true);
6906 LD->getPointerInfo().getWithOffset(8),
LD->getOriginalAlign(),
6907 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
6909 DCI.CombineTo(LoPart, EltLoad,
true);
6916 DCI.AddToWorklist(Chain.
getNode());
6938 else if (UI.getUse().getResNo() == 0)
6941 if (!Replicate || OtherUses.
empty())
6947 for (
SDNode *U : OtherUses) {
6956bool SystemZTargetLowering::canLoadStoreByteSwapped(
EVT VT)
const {
6957 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64)
6959 if (Subtarget.hasVectorEnhancements2())
6960 if (VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v2i64 || VT == MVT::i128)
6972 for (
unsigned i = 0; i < NumElts; ++i) {
6973 if (M[i] < 0)
continue;
6974 if ((
unsigned) M[i] != NumElts - 1 - i)
6982 for (
auto *U : StoredVal->
uses()) {
6984 EVT CurrMemVT = ST->getMemoryVT().getScalarType();
6987 }
else if (isa<BuildVectorSDNode>(U)) {
7043SDValue SystemZTargetLowering::combineSTORE(
7044 SDNode *
N, DAGCombinerInfo &DCI)
const {
7046 auto *SN = cast<StoreSDNode>(
N);
7047 auto &Op1 =
N->getOperand(1);
7048 EVT MemVT = SN->getMemoryVT();
7053 if (MemVT.
isInteger() && SN->isTruncatingStore()) {
7055 combineTruncateExtract(
SDLoc(
N), MemVT, SN->getValue(), DCI)) {
7056 DCI.AddToWorklist(
Value.getNode());
7060 SN->getBasePtr(), SN->getMemoryVT(),
7061 SN->getMemOperand());
7065 if (!SN->isTruncatingStore() &&
7076 N->getOperand(0), BSwapOp,
N->getOperand(2)
7081 Ops, MemVT, SN->getMemOperand());
7084 if (!SN->isTruncatingStore() &&
7087 Subtarget.hasVectorEnhancements2()) {
7097 Ops, MemVT, SN->getMemOperand());
7102 if (!SN->isTruncatingStore() &&
7105 N->getOperand(0).reachesChainWithoutSideEffects(
SDValue(Op1.
getNode(), 1))) {
7109 Ops, MemVT, SN->getMemOperand());
7119 DAG.
getStore(SN->getChain(),
DL, HiPart, SN->getBasePtr(),
7120 SN->getPointerInfo(), SN->getOriginalAlign(),
7121 SN->getMemOperand()->getFlags(), SN->getAAInfo());
7126 SN->getPointerInfo().getWithOffset(8),
7127 SN->getOriginalAlign(),
7128 SN->getMemOperand()->
getFlags(), SN->getAAInfo());
7148 if (
C->getAPIntValue().getBitWidth() > 64 ||
C->isAllOnes() ||
7152 if (VCI.isVectorConstantLegal(Subtarget) &&
7161 auto FindReplicatedReg = [&](
SDValue MulOp) {
7162 EVT MulVT = MulOp.getValueType();
7163 if (MulOp->getOpcode() ==
ISD::MUL &&
7164 (MulVT == MVT::i16 || MulVT == MVT::i32 || MulVT == MVT::i64)) {
7168 WordVT =
LHS->getOperand(0).getValueType();
7170 WordVT = cast<VTSDNode>(
LHS->getOperand(1))->getVT();
7174 if (
auto *
C = dyn_cast<ConstantSDNode>(MulOp->getOperand(1))) {
7176 APInt(MulVT.getSizeInBits(),
C->getZExtValue()));
7177 if (VCI.isVectorConstantLegal(Subtarget) &&
7179 WordVT == VCI.VecVT.getScalarType())
7185 if (isa<BuildVectorSDNode>(Op1) &&
7188 if (
auto *
C = dyn_cast<ConstantSDNode>(SplatVal))
7191 FindReplicatedReg(SplatVal);
7193 if (
auto *
C = dyn_cast<ConstantSDNode>(Op1))
7196 FindReplicatedReg(Op1);
7201 "Bad type handling");
7206 SN->getBasePtr(), SN->getMemOperand());
7213SDValue SystemZTargetLowering::combineVECTOR_SHUFFLE(
7214 SDNode *
N, DAGCombinerInfo &DCI)
const {
7218 N->getOperand(0).hasOneUse() &&
7219 Subtarget.hasVectorEnhancements2()) {
7234 Ops,
LD->getMemoryVT(),
LD->getMemOperand());
7238 DCI.CombineTo(
N, ESLoad);
7242 DCI.CombineTo(
Load.getNode(), ESLoad, ESLoad.
getValue(1));
7252SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
7253 SDNode *
N, DAGCombinerInfo &DCI)
const {
7256 if (!Subtarget.hasVector())
7262 Op.getValueType().isVector() &&
7263 Op.getOperand(0).getValueType().isVector() &&
7264 Op.getValueType().getVectorNumElements() ==
7265 Op.getOperand(0).getValueType().getVectorNumElements())
7266 Op =
Op.getOperand(0);
7270 EVT VecVT =
Op.getValueType();
7273 Op.getOperand(0),
N->getOperand(1));
7274 DCI.AddToWorklist(
Op.getNode());
7276 if (EltVT !=
N->getValueType(0)) {
7277 DCI.AddToWorklist(
Op.getNode());
7284 if (
auto *IndexN = dyn_cast<ConstantSDNode>(
N->getOperand(1))) {
7287 return combineExtract(
SDLoc(
N),
N->getValueType(0), VecVT, Op0,
7288 IndexN->getZExtValue(), DCI,
false);
7293SDValue SystemZTargetLowering::combineJOIN_DWORDS(
7294 SDNode *
N, DAGCombinerInfo &DCI)
const {
7297 if (
N->getOperand(0) ==
N->getOperand(1))
7308 if (Chain1 == Chain2)
7316SDValue SystemZTargetLowering::combineFP_ROUND(
7317 SDNode *
N, DAGCombinerInfo &DCI)
const {
7319 if (!Subtarget.hasVector())
7328 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
7331 if (
N->getValueType(0) == MVT::f32 && Op0.
hasOneUse() &&
7337 for (
auto *U : Vec->
uses()) {
7338 if (U != Op0.
getNode() &&
U->hasOneUse() &&
7340 U->getOperand(0) == Vec &&
7342 U->getConstantOperandVal(1) == 1) {
7344 if (OtherRound.
getOpcode() ==
N->getOpcode() &&
7348 if (
N->isStrictFPOpcode()) {
7353 {MVT::v4f32, MVT::Other}, {Chain, Vec});
7358 DCI.AddToWorklist(VRound.
getNode());
7362 DCI.AddToWorklist(Extract1.
getNode());
7371 N->getVTList(), Extract0, Chain);
7380SDValue SystemZTargetLowering::combineFP_EXTEND(
7381 SDNode *
N, DAGCombinerInfo &DCI)
const {
7383 if (!Subtarget.hasVector())
7392 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
7395 if (
N->getValueType(0) == MVT::f64 && Op0.
hasOneUse() &&
7401 for (
auto *U : Vec->
uses()) {
7402 if (U != Op0.
getNode() &&
U->hasOneUse() &&
7404 U->getOperand(0) == Vec &&
7406 U->getConstantOperandVal(1) == 2) {
7408 if (OtherExtend.
getOpcode() ==
N->getOpcode() &&
7412 if (
N->isStrictFPOpcode()) {
7417 {MVT::v2f64, MVT::Other}, {Chain, Vec});
7422 DCI.AddToWorklist(VExtend.
getNode());
7426 DCI.AddToWorklist(Extract1.
getNode());
7435 N->getVTList(), Extract0, Chain);
7444SDValue SystemZTargetLowering::combineINT_TO_FP(
7445 SDNode *
N, DAGCombinerInfo &DCI)
const {
7450 unsigned Opcode =
N->getOpcode();
7451 EVT OutVT =
N->getValueType(0);
7455 unsigned InScalarBits =
Op->getValueType(0).getScalarSizeInBits();
7461 if (OutLLVMTy->
isVectorTy() && OutScalarBits > InScalarBits &&
7462 OutScalarBits <= 64) {
7463 unsigned NumElts = cast<FixedVectorType>(OutLLVMTy)->getNumElements();
7466 unsigned ExtOpcode =
7474SDValue SystemZTargetLowering::combineBSWAP(
7475 SDNode *
N, DAGCombinerInfo &DCI)
const {
7479 N->getOperand(0).hasOneUse() &&
7480 canLoadStoreByteSwapped(
N->getValueType(0))) {
7489 EVT LoadVT =
N->getValueType(0);
7490 if (LoadVT == MVT::i16)
7495 Ops,
LD->getMemoryVT(),
LD->getMemOperand());
7499 if (
N->getValueType(0) == MVT::i16)
7504 DCI.CombineTo(
N, ResVal);
7508 DCI.CombineTo(
Load.getNode(), ResVal, BSLoad.
getValue(1));
7517 Op.getValueType().isVector() &&
7518 Op.getOperand(0).getValueType().isVector() &&
7519 Op.getValueType().getVectorNumElements() ==
7520 Op.getOperand(0).getValueType().getVectorNumElements())
7521 Op =
Op.getOperand(0);
7533 (canLoadStoreByteSwapped(
N->getValueType(0)) &&
7535 EVT VecVT =
N->getValueType(0);
7536 EVT EltVT =
N->getValueType(0).getVectorElementType();
7539 DCI.AddToWorklist(Vec.
getNode());
7543 DCI.AddToWorklist(Elt.
getNode());
7546 DCI.AddToWorklist(Vec.
getNode());
7548 DCI.AddToWorklist(Elt.
getNode());
7556 if (SV &&
Op.hasOneUse()) {
7564 EVT VecVT =
N->getValueType(0);
7567 DCI.AddToWorklist(Op0.
getNode());
7571 DCI.AddToWorklist(Op1.
getNode());
7574 DCI.AddToWorklist(Op0.
getNode());
7576 DCI.AddToWorklist(Op1.
getNode());
7598 auto *CompareRHS = dyn_cast<ConstantSDNode>(ICmp->getOperand(1));
7605 bool Invert =
false;
7612 auto *TrueVal = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(0));
7615 auto *FalseVal = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(1));
7618 if (CompareRHS->getZExtValue() == FalseVal->getZExtValue())
7620 else if (CompareRHS->getZExtValue() != TrueVal->getZExtValue())
7624 auto *NewCCValid = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(2));
7625 auto *NewCCMask = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(3));
7626 if (!NewCCValid || !NewCCMask)
7628 CCValid = NewCCValid->getZExtValue();
7629 CCMask = NewCCMask->getZExtValue();
7639 if (CompareLHS->getOpcode() ==
ISD::SRA) {
7640 auto *SRACount = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(1));
7641 if (!SRACount || SRACount->getZExtValue() != 30)
7643 auto *SHL = CompareLHS->getOperand(0).getNode();
7646 auto *SHLCount = dyn_cast<ConstantSDNode>(SHL->getOperand(1));
7649 auto *IPM = SHL->getOperand(0).getNode();
7654 if (!CompareLHS->hasOneUse())
7657 if (CompareRHS->getZExtValue() != 0)
7664 CCReg = IPM->getOperand(0);
7671SDValue SystemZTargetLowering::combineBR_CCMASK(
7672 SDNode *
N, DAGCombinerInfo &DCI)
const {
7676 auto *CCValid = dyn_cast<ConstantSDNode>(
N->getOperand(1));
7677 auto *CCMask = dyn_cast<ConstantSDNode>(
N->getOperand(2));
7678 if (!CCValid || !CCMask)
7681 int CCValidVal = CCValid->getZExtValue();
7682 int CCMaskVal = CCMask->getZExtValue();
7691 N->getOperand(3), CCReg);
7695SDValue SystemZTargetLowering::combineSELECT_CCMASK(
7696 SDNode *
N, DAGCombinerInfo &DCI)
const {
7700 auto *CCValid = dyn_cast<ConstantSDNode>(
N->getOperand(2));
7701 auto *CCMask = dyn_cast<ConstantSDNode>(
N->getOperand(3));
7702 if (!CCValid || !CCMask)
7705 int CCValidVal = CCValid->getZExtValue();
7706 int CCMaskVal = CCMask->getZExtValue();
7711 N->getOperand(0),
N->getOperand(1),
7719SDValue SystemZTargetLowering::combineGET_CCMASK(
7720 SDNode *
N, DAGCombinerInfo &DCI)
const {
7723 auto *CCValid = dyn_cast<ConstantSDNode>(
N->getOperand(1));
7724 auto *CCMask = dyn_cast<ConstantSDNode>(
N->getOperand(2));
7725 if (!CCValid || !CCMask)
7727 int CCValidVal = CCValid->getZExtValue();
7728 int CCMaskVal = CCMask->getZExtValue();
7736 auto *SelectCCValid = dyn_cast<ConstantSDNode>(
Select->getOperand(2));
7737 auto *SelectCCMask = dyn_cast<ConstantSDNode>(
Select->getOperand(3));
7738 if (!SelectCCValid || !SelectCCMask)
7740 int SelectCCValidVal = SelectCCValid->getZExtValue();
7741 int SelectCCMaskVal = SelectCCMask->getZExtValue();
7743 auto *
TrueVal = dyn_cast<ConstantSDNode>(
Select->getOperand(0));
7744 auto *
FalseVal = dyn_cast<ConstantSDNode>(
Select->getOperand(1));
7745 if (!TrueVal || !FalseVal)
7749 else if (
TrueVal->getZExtValue() == 0 &&
FalseVal->getZExtValue() == 1)
7750 SelectCCMaskVal ^= SelectCCValidVal;
7754 if (SelectCCValidVal & ~CCValidVal)
7756 if (SelectCCMaskVal != (CCMaskVal & SelectCCValidVal))
7759 return Select->getOperand(4);
7762SDValue SystemZTargetLowering::combineIntDIVREM(
7763 SDNode *
N, DAGCombinerInfo &DCI)
const {
7765 EVT VT =
N->getValueType(0);
7779SDValue SystemZTargetLowering::combineINTRINSIC(
7780 SDNode *
N, DAGCombinerInfo &DCI)
const {
7783 unsigned Id =
N->getConstantOperandVal(1);
7787 case Intrinsic::s390_vll:
7788 case Intrinsic::s390_vlrl:
7789 if (
auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(2)))
7790 if (
C->getZExtValue() >= 15)
7795 case Intrinsic::s390_vstl:
7796 case Intrinsic::s390_vstrl:
7797 if (
auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(3)))
7798 if (
C->getZExtValue() >= 15)
7809 return N->getOperand(0);
7815 switch(
N->getOpcode()) {
7840 case ISD::UREM:
return combineIntDIVREM(
N, DCI);
7852 EVT VT =
Op.getValueType();
7855 unsigned Opcode =
Op.getOpcode();
7857 unsigned Id =
Op.getConstantOperandVal(0);
7859 case Intrinsic::s390_vpksh:
7860 case Intrinsic::s390_vpksf:
7861 case Intrinsic::s390_vpksg:
7862 case Intrinsic::s390_vpkshs:
7863 case Intrinsic::s390_vpksfs:
7864 case Intrinsic::s390_vpksgs:
7865 case Intrinsic::s390_vpklsh:
7866 case Intrinsic::s390_vpklsf:
7867 case Intrinsic::s390_vpklsg:
7868 case Intrinsic::s390_vpklshs:
7869 case Intrinsic::s390_vpklsfs:
7870 case Intrinsic::s390_vpklsgs:
7872 SrcDemE = DemandedElts;
7875 SrcDemE = SrcDemE.
trunc(NumElts / 2);
7878 case Intrinsic::s390_vuphb:
7879 case Intrinsic::s390_vuphh:
7880 case Intrinsic::s390_vuphf:
7881 case Intrinsic::s390_vuplhb:
7882 case Intrinsic::s390_vuplhh:
7883 case Intrinsic::s390_vuplhf:
7884 SrcDemE =
APInt(NumElts * 2, 0);
7887 case Intrinsic::s390_vuplb:
7888 case Intrinsic::s390_vuplhw:
7889 case Intrinsic::s390_vuplf:
7890 case Intrinsic::s390_vupllb:
7891 case Intrinsic::s390_vupllh:
7892 case Intrinsic::s390_vupllf:
7893 SrcDemE =
APInt(NumElts * 2, 0);
7896 case Intrinsic::s390_vpdi: {
7898 SrcDemE =
APInt(NumElts, 0);
7899 if (!DemandedElts[OpNo - 1])
7901 unsigned Mask =
Op.getConstantOperandVal(3);
7902 unsigned MaskBit = ((OpNo - 1) ? 1 : 4);
7904 SrcDemE.
setBit((Mask & MaskBit)? 1 : 0);
7907 case Intrinsic::s390_vsldb: {
7909 assert(VT == MVT::v16i8 &&
"Unexpected type.");
7910 unsigned FirstIdx =
Op.getConstantOperandVal(3);
7911 assert (FirstIdx > 0 && FirstIdx < 16 &&
"Unused operand.");
7912 unsigned NumSrc0Els = 16 - FirstIdx;
7913 SrcDemE =
APInt(NumElts, 0);
7915 APInt DemEls = DemandedElts.
trunc(NumSrc0Els);
7918 APInt DemEls = DemandedElts.
lshr(NumSrc0Els);
7923 case Intrinsic::s390_vperm:
7924 SrcDemE =
APInt(NumElts, -1);
7934 SrcDemE =
APInt(1, 1);
7937 SrcDemE = DemandedElts;
7948 const APInt &DemandedElts,
7963 const APInt &DemandedElts,
7965 unsigned Depth)
const {
7969 unsigned tmp0, tmp1;
7974 EVT VT =
Op.getValueType();
7975 if (
Op.getResNo() != 0 || VT == MVT::Untyped)
7978 "KnownBits does not match VT in bitwidth");
7981 "DemandedElts does not match VT number of elements");
7983 unsigned Opcode =
Op.getOpcode();
7985 bool IsLogical =
false;
7986 unsigned Id =
Op.getConstantOperandVal(0);
7988 case Intrinsic::s390_vpksh:
7989 case Intrinsic::s390_vpksf:
7990 case Intrinsic::s390_vpksg:
7991 case Intrinsic::s390_vpkshs:
7992 case Intrinsic::s390_vpksfs:
7993 case Intrinsic::s390_vpksgs:
7994 case Intrinsic::s390_vpklsh:
7995 case Intrinsic::s390_vpklsf:
7996 case Intrinsic::s390_vpklsg:
7997 case Intrinsic::s390_vpklshs:
7998 case Intrinsic::s390_vpklsfs:
7999 case Intrinsic::s390_vpklsgs:
8000 case Intrinsic::s390_vpdi:
8001 case Intrinsic::s390_vsldb:
8002 case Intrinsic::s390_vperm:
8005 case Intrinsic::s390_vuplhb:
8006 case Intrinsic::s390_vuplhh:
8007 case Intrinsic::s390_vuplhf:
8008 case Intrinsic::s390_vupllb:
8009 case Intrinsic::s390_vupllh:
8010 case Intrinsic::s390_vupllf:
8013 case Intrinsic::s390_vuphb:
8014 case Intrinsic::s390_vuphh:
8015 case Intrinsic::s390_vuphf:
8016 case Intrinsic::s390_vuplb:
8017 case Intrinsic::s390_vuplhw:
8018 case Intrinsic::s390_vuplf: {
8060 if (
LHS == 1)
return 1;
8063 if (
RHS == 1)
return 1;
8064 unsigned Common = std::min(
LHS,
RHS);
8065 unsigned SrcBitWidth =
Op.getOperand(OpNo).getScalarValueSizeInBits();
8066 EVT VT =
Op.getValueType();
8068 if (SrcBitWidth > VTBits) {
8069 unsigned SrcExtraBits = SrcBitWidth - VTBits;
8070 if (Common > SrcExtraBits)
8071 return (Common - SrcExtraBits);
8074 assert (SrcBitWidth == VTBits &&
"Expected operands of same bitwidth.");
8081 unsigned Depth)
const {
8082 if (
Op.getResNo() != 0)
8084 unsigned Opcode =
Op.getOpcode();
8086 unsigned Id =
Op.getConstantOperandVal(0);
8088 case Intrinsic::s390_vpksh:
8089 case Intrinsic::s390_vpksf:
8090 case Intrinsic::s390_vpksg:
8091 case Intrinsic::s390_vpkshs:
8092 case Intrinsic::s390_vpksfs:
8093 case Intrinsic::s390_vpksgs:
8094 case Intrinsic::s390_vpklsh:
8095 case Intrinsic::s390_vpklsf:
8096 case Intrinsic::s390_vpklsg:
8097 case Intrinsic::s390_vpklshs:
8098 case Intrinsic::s390_vpklsfs:
8099 case Intrinsic::s390_vpklsgs:
8100 case Intrinsic::s390_vpdi:
8101 case Intrinsic::s390_vsldb:
8102 case Intrinsic::s390_vperm:
8104 case Intrinsic::s390_vuphb:
8105 case Intrinsic::s390_vuphh:
8106 case Intrinsic::s390_vuphf:
8107 case Intrinsic::s390_vuplb:
8108 case Intrinsic::s390_vuplhw:
8109 case Intrinsic::s390_vuplf: {
8113 EVT VT =
Op.getValueType();
8137 switch (
Op->getOpcode()) {
8150 "Unexpected stack alignment");
8153 unsigned StackProbeSize =
8156 StackProbeSize &= ~(StackAlign - 1);
8157 return StackProbeSize ? StackProbeSize : StackAlign;
8174 Register Reg =
MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
8180 Register Reg =
MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
8206 if (Succ->isLiveIn(SystemZ::CC))
8217 switch (
MI.getOpcode()) {
8218 case SystemZ::Select32:
8219 case SystemZ::Select64:
8220 case SystemZ::Select128:
8221 case SystemZ::SelectF32:
8222 case SystemZ::SelectF64:
8223 case SystemZ::SelectF128:
8224 case SystemZ::SelectVR32:
8225 case SystemZ::SelectVR64:
8226 case SystemZ::SelectVR128:
8258 for (
auto *
MI : Selects) {
8259 Register DestReg =
MI->getOperand(0).getReg();
8260 Register TrueReg =
MI->getOperand(1).getReg();
8261 Register FalseReg =
MI->getOperand(2).getReg();
8266 if (
MI->getOperand(4).getImm() == (CCValid ^ CCMask))
8269 if (RegRewriteTable.
contains(TrueReg))
8270 TrueReg = RegRewriteTable[TrueReg].first;
8272 if (RegRewriteTable.
contains(FalseReg))
8273 FalseReg = RegRewriteTable[FalseReg].second;
8276 BuildMI(*SinkMBB, SinkInsertionPoint,
DL,
TII->get(SystemZ::PHI), DestReg)
8281 RegRewriteTable[DestReg] = std::make_pair(TrueReg, FalseReg);
8293 assert(TFL->hasReservedCallFrame(MF) &&
8294 "ADJSTACKDOWN and ADJSTACKUP should be no-ops");
8299 uint32_t NumBytes =
MI.getOperand(0).getImm();
8304 MI.eraseFromParent();
8315 unsigned CCValid =
MI.getOperand(3).getImm();
8316 unsigned CCMask =
MI.getOperand(4).getImm();
8328 assert(NextMI.getOperand(3).getImm() == CCValid &&
8329 "Bad CCValid operands since CC was not redefined.");
8330 if (NextMI.getOperand(4).getImm() == CCMask ||
8331 NextMI.getOperand(4).getImm() == (CCValid ^ CCMask)) {
8337 if (NextMI.definesRegister(SystemZ::CC,
nullptr) ||
8338 NextMI.usesCustomInsertionHook())
8341 for (
auto *SelMI : Selects)
8342 if (NextMI.readsVirtualRegister(SelMI->getOperand(0).getReg())) {
8346 if (NextMI.isDebugInstr()) {
8348 assert(NextMI.isDebugValue() &&
"Unhandled debug opcode.");
8351 }
else if (
User || ++Count > 20)
8356 bool CCKilled = (LastMI->
killsRegister(SystemZ::CC,
nullptr) ||
8388 for (
auto *SelMI : Selects)
8389 SelMI->eraseFromParent();
8392 for (
auto *DbgMI : DbgValues)
8393 MBB->
splice(InsertPos, StartMBB, DbgMI);
8404 unsigned StoreOpcode,
8405 unsigned STOCOpcode,
8406 bool Invert)
const {
8411 int64_t Disp =
MI.getOperand(2).getImm();
8412 Register IndexReg =
MI.getOperand(3).getReg();
8413 unsigned CCValid =
MI.getOperand(4).getImm();
8414 unsigned CCMask =
MI.getOperand(5).getImm();
8417 StoreOpcode =
TII->getOpcodeForOffset(StoreOpcode, Disp);
8422 for (
auto *
I :
MI.memoperands())
8431 if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) {
8443 MI.eraseFromParent();
8457 if (!
MI.killsRegister(SystemZ::CC,
nullptr) &&
8484 MI.eraseFromParent();
8520 int HiOpcode =
Unsigned? SystemZ::VECLG : SystemZ::VECG;
8539 Register Temp =
MRI.createVirtualRegister(&SystemZ::VR128BitRegClass);
8547 MI.eraseFromParent();
8558 bool Invert)
const {
8567 int64_t Disp =
MI.getOperand(2).getImm();
8569 Register BitShift =
MI.getOperand(4).getReg();
8570 Register NegBitShift =
MI.getOperand(5).getReg();
8571 unsigned BitSize =
MI.getOperand(6).getImm();
8575 unsigned LOpcode =
TII->getOpcodeForOffset(SystemZ::L, Disp);
8576 unsigned CSOpcode =
TII->getOpcodeForOffset(SystemZ::CS, Disp);
8577 assert(LOpcode && CSOpcode &&
"Displacement out of range");
8580 Register OrigVal =
MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8581 Register OldVal =
MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8582 Register NewVal =
MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8583 Register RotatedOldVal =
MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8584 Register RotatedNewVal =
MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8615 Register Tmp =
MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8620 }
else if (BinOpcode)
8643 MI.eraseFromParent();
8654 unsigned KeepOldMask)
const {
8662 int64_t Disp =
MI.getOperand(2).getImm();
8664 Register BitShift =
MI.getOperand(4).getReg();
8665 Register NegBitShift =
MI.getOperand(5).getReg();
8666 unsigned BitSize =
MI.getOperand(6).getImm();
8670 unsigned LOpcode =
TII->getOpcodeForOffset(SystemZ::L, Disp);
8671 unsigned CSOpcode =
TII->getOpcodeForOffset(SystemZ::CS, Disp);
8672 assert(LOpcode && CSOpcode &&
"Displacement out of range");
8675 Register OrigVal =
MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8676 Register OldVal =
MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8677 Register NewVal =
MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8678 Register RotatedOldVal =
MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8679 Register RotatedAltVal =
MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8680 Register RotatedNewVal =
MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8747 MI.eraseFromParent();
8763 int64_t Disp =
MI.getOperand(2).getImm();
8765 Register OrigSwapVal =
MI.getOperand(4).getReg();
8766 Register BitShift =
MI.getOperand(5).getReg();
8767 Register NegBitShift =
MI.getOperand(6).getReg();
8768 int64_t BitSize =
MI.getOperand(7).getImm();
8774 unsigned LOpcode =
TII->getOpcodeForOffset(SystemZ::L, Disp);
8775 unsigned CSOpcode =
TII->getOpcodeForOffset(SystemZ::CS, Disp);
8776 unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
8777 assert(LOpcode && CSOpcode &&
"Displacement out of range");
8780 Register OrigOldVal =
MRI.createVirtualRegister(RC);
8783 Register StoreVal =
MRI.createVirtualRegister(RC);
8784 Register OldValRot =
MRI.createVirtualRegister(RC);
8785 Register RetryOldVal =
MRI.createVirtualRegister(RC);
8786 Register RetrySwapVal =
MRI.createVirtualRegister(RC);
8861 if (!
MI.registerDefIsDead(SystemZ::CC,
nullptr))
8864 MI.eraseFromParent();
8877 .
add(
MI.getOperand(1))
8878 .
addImm(SystemZ::subreg_h64)
8879 .
add(
MI.getOperand(2))
8880 .
addImm(SystemZ::subreg_l64);
8881 MI.eraseFromParent();
8890 bool ClearEven)
const {
8898 Register In128 =
MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
8902 Register NewIn128 =
MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
8903 Register Zero64 =
MRI.createVirtualRegister(&SystemZ::GR64BitRegClass);
8914 MI.eraseFromParent();
8921 unsigned Opcode,
bool IsMemset)
const {
8928 uint64_t DestDisp =
MI.getOperand(1).getImm();
8934 if (!isUInt<12>(Disp)) {
8935 Register Reg =
MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
8936 unsigned Opcode =
TII->getOpcodeForOffset(SystemZ::LA, Disp);
8946 SrcDisp =
MI.getOperand(3).getImm();
8949 SrcDisp = DestDisp++;
8950 foldDisplIfNeeded(DestBase, DestDisp);
8954 bool IsImmForm = LengthMO.
isImm();
8955 bool IsRegForm = !IsImmForm;
8962 unsigned Length) ->
void {
8981 bool NeedsLoop =
false;
8983 Register LenAdjReg = SystemZ::NoRegister;
8985 ImmLength = LengthMO.
getImm();
8986 ImmLength += IsMemset ? 2 : 1;
8987 if (ImmLength == 0) {
8988 MI.eraseFromParent();
8991 if (Opcode == SystemZ::CLC) {
8992 if (ImmLength > 3 * 256)
9002 }
else if (ImmLength > 6 * 256)
9010 LenAdjReg = LengthMO.
getReg();
9016 (Opcode == SystemZ::CLC && (ImmLength > 256 || NeedsLoop)
9022 MRI.createVirtualRegister(&SystemZ::GR64BitRegClass);
9024 TII->loadImmediate(*
MBB,
MI, StartCountReg, ImmLength / 256);
9035 Register Reg =
MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9039 if (DestBase.
isReg() && DestBase.
getReg() == SystemZ::NoRegister)
9040 DestBase = loadZeroAddress();
9041 if (SrcBase.
isReg() && SrcBase.
getReg() == SystemZ::NoRegister)
9042 SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
9052 (HaveSingleBase ? StartSrcReg :
forceReg(
MI, DestBase,
TII));
9055 Register ThisSrcReg =
MRI.createVirtualRegister(RC);
9057 (HaveSingleBase ? ThisSrcReg :
MRI.createVirtualRegister(RC));
9058 Register NextSrcReg =
MRI.createVirtualRegister(RC);
9060 (HaveSingleBase ? NextSrcReg :
MRI.createVirtualRegister(RC));
9061 RC = &SystemZ::GR64BitRegClass;
9062 Register ThisCountReg =
MRI.createVirtualRegister(RC);
9063 Register NextCountReg =
MRI.createVirtualRegister(RC);
9089 MBB = MemsetOneCheckMBB;
9132 if (EndMBB && !ImmLength)
9154 if (!HaveSingleBase)
9161 if (Opcode == SystemZ::MVC)
9188 if (!HaveSingleBase)
9210 Register RemSrcReg =
MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9211 Register RemDestReg = HaveSingleBase ? RemSrcReg
9212 :
MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9216 if (!HaveSingleBase)
9232 if (Opcode != SystemZ::MVC) {
9242 while (ImmLength > 0) {
9246 foldDisplIfNeeded(DestBase, DestDisp);
9247 foldDisplIfNeeded(SrcBase, SrcDisp);
9248 insertMemMemOp(
MBB,
MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength);
9249 DestDisp += ThisLength;
9250 SrcDisp += ThisLength;
9251 ImmLength -= ThisLength;
9254 if (EndMBB && ImmLength > 0) {
9270 MI.eraseFromParent();
9283 uint64_t End1Reg =
MI.getOperand(0).getReg();
9284 uint64_t Start1Reg =
MI.getOperand(1).getReg();
9285 uint64_t Start2Reg =
MI.getOperand(2).getReg();
9286 uint64_t CharReg =
MI.getOperand(3).getReg();
9289 uint64_t This1Reg =
MRI.createVirtualRegister(RC);
9290 uint64_t This2Reg =
MRI.createVirtualRegister(RC);
9329 MI.eraseFromParent();
9336 bool NoFloat)
const {
9342 MI.setDesc(
TII->get(Opcode));
9346 uint64_t Control =
MI.getOperand(2).getImm();
9347 static const unsigned GPRControlBit[16] = {
9348 0x8000, 0x8000, 0x4000, 0x4000, 0x2000, 0x2000, 0x1000, 0x1000,
9349 0x0800, 0x0800, 0x0400, 0x0400, 0x0200, 0x0200, 0x0100, 0x0100
9351 Control |= GPRControlBit[15];
9353 Control |= GPRControlBit[11];
9354 MI.getOperand(2).setImm(Control);
9357 for (
int I = 0;
I < 16;
I++) {
9358 if ((Control & GPRControlBit[
I]) == 0) {
9365 if (!NoFloat && (Control & 4) != 0) {
9366 if (Subtarget.hasVector()) {
9398 MI.eraseFromParent();
9411 Register SizeReg =
MI.getOperand(2).getReg();
9423 Register PHIReg =
MRI->createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9424 Register IncReg =
MRI->createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9489 MI.eraseFromParent();
9493SDValue SystemZTargetLowering::
9504 switch (
MI.getOpcode()) {
9505 case SystemZ::ADJCALLSTACKDOWN:
9506 case SystemZ::ADJCALLSTACKUP:
9507 return emitAdjCallStack(
MI,
MBB);
9509 case SystemZ::Select32:
9510 case SystemZ::Select64:
9511 case SystemZ::Select128:
9512 case SystemZ::SelectF32:
9513 case SystemZ::SelectF64:
9514 case SystemZ::SelectF128:
9515 case SystemZ::SelectVR32:
9516 case SystemZ::SelectVR64:
9517 case SystemZ::SelectVR128:
9518 return emitSelect(
MI,
MBB);
9520 case SystemZ::CondStore8Mux:
9521 return emitCondStore(
MI,
MBB, SystemZ::STCMux, 0,
false);
9522 case SystemZ::CondStore8MuxInv:
9523 return emitCondStore(
MI,
MBB, SystemZ::STCMux, 0,
true);
9524 case SystemZ::CondStore16Mux:
9525 return emitCondStore(
MI,
MBB, SystemZ::STHMux, 0,
false);
9526 case SystemZ::CondStore16MuxInv:
9527 return emitCondStore(
MI,
MBB, SystemZ::STHMux, 0,
true);
9528 case SystemZ::CondStore32Mux:
9529 return emitCondStore(
MI,
MBB, SystemZ::STMux, SystemZ::STOCMux,
false);
9530 case SystemZ::CondStore32MuxInv:
9531 return emitCondStore(
MI,
MBB, SystemZ::STMux, SystemZ::STOCMux,
true);
9532 case SystemZ::CondStore8:
9533 return emitCondStore(
MI,
MBB, SystemZ::STC, 0,
false);
9534 case SystemZ::CondStore8Inv:
9535 return emitCondStore(
MI,
MBB, SystemZ::STC, 0,
true);
9536 case SystemZ::CondStore16:
9537 return emitCondStore(
MI,
MBB, SystemZ::STH, 0,
false);
9538 case SystemZ::CondStore16Inv:
9539 return emitCondStore(
MI,
MBB, SystemZ::STH, 0,
true);
9540 case SystemZ::CondStore32:
9541 return emitCondStore(
MI,
MBB, SystemZ::ST, SystemZ::STOC,
false);
9542 case SystemZ::CondStore32Inv:
9543 return emitCondStore(
MI,
MBB, SystemZ::ST, SystemZ::STOC,
true);
9544 case SystemZ::CondStore64:
9545 return emitCondStore(
MI,
MBB, SystemZ::STG, SystemZ::STOCG,
false);
9546 case SystemZ::CondStore64Inv:
9547 return emitCondStore(
MI,
MBB, SystemZ::STG, SystemZ::STOCG,
true);
9548 case SystemZ::CondStoreF32:
9549 return emitCondStore(
MI,
MBB, SystemZ::STE, 0,
false);
9550 case SystemZ::CondStoreF32Inv:
9551 return emitCondStore(
MI,
MBB, SystemZ::STE, 0,
true);
9552 case SystemZ::CondStoreF64:
9553 return emitCondStore(
MI,
MBB, SystemZ::STD, 0,
false);
9554 case SystemZ::CondStoreF64Inv:
9555 return emitCondStore(
MI,
MBB, SystemZ::STD, 0,
true);
9557 case SystemZ::SCmp128Hi:
9558 return emitICmp128Hi(
MI,
MBB,
false);
9559 case SystemZ::UCmp128Hi:
9560 return emitICmp128Hi(
MI,
MBB,
true);
9562 case SystemZ::PAIR128:
9563 return emitPair128(
MI,
MBB);
9564 case SystemZ::AEXT128:
9565 return emitExt128(
MI,
MBB,
false);
9566 case SystemZ::ZEXT128:
9567 return emitExt128(
MI,
MBB,
true);
9569 case SystemZ::ATOMIC_SWAPW:
9570 return emitAtomicLoadBinary(
MI,
MBB, 0);
9572 case SystemZ::ATOMIC_LOADW_AR:
9573 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::AR);
9574 case SystemZ::ATOMIC_LOADW_AFI:
9575 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::AFI);
9577 case SystemZ::ATOMIC_LOADW_SR:
9578 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::SR);
9580 case SystemZ::ATOMIC_LOADW_NR:
9581 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::NR);
9582 case SystemZ::ATOMIC_LOADW_NILH:
9583 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::NILH);
9585 case SystemZ::ATOMIC_LOADW_OR:
9586 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::OR);
9587 case SystemZ::ATOMIC_LOADW_OILH:
9588 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::OILH);
9590 case SystemZ::ATOMIC_LOADW_XR:
9591 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::XR);
9592 case SystemZ::ATOMIC_LOADW_XILF:
9593 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::XILF);
9595 case SystemZ::ATOMIC_LOADW_NRi:
9596 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::NR,
true);
9597 case SystemZ::ATOMIC_LOADW_NILHi:
9598 return emitAtomicLoadBinary(
MI,
MBB, SystemZ::NILH,
true);
9600 case SystemZ::ATOMIC_LOADW_MIN:
9602 case SystemZ::ATOMIC_LOADW_MAX:
9604 case SystemZ::ATOMIC_LOADW_UMIN:
9606 case SystemZ::ATOMIC_LOADW_UMAX:
9609 case SystemZ::ATOMIC_CMP_SWAPW:
9610 return emitAtomicCmpSwapW(
MI,
MBB);
9611 case SystemZ::MVCImm:
9612 case SystemZ::MVCReg:
9613 return emitMemMemWrapper(
MI,
MBB, SystemZ::MVC);
9614 case SystemZ::NCImm:
9615 return emitMemMemWrapper(
MI,
MBB, SystemZ::NC);
9616 case SystemZ::OCImm:
9617 return emitMemMemWrapper(
MI,
MBB, SystemZ::OC);
9618 case SystemZ::XCImm:
9619 case SystemZ::XCReg:
9620 return emitMemMemWrapper(
MI,
MBB, SystemZ::XC);
9621 case SystemZ::CLCImm:
9622 case SystemZ::CLCReg:
9623 return emitMemMemWrapper(
MI,
MBB, SystemZ::CLC);
9624 case SystemZ::MemsetImmImm:
9625 case SystemZ::MemsetImmReg:
9626 case SystemZ::MemsetRegImm:
9627 case SystemZ::MemsetRegReg:
9628 return emitMemMemWrapper(
MI,
MBB, SystemZ::MVC,
true);
9629 case SystemZ::CLSTLoop:
9630 return emitStringWrapper(
MI,
MBB, SystemZ::CLST);
9631 case SystemZ::MVSTLoop:
9632 return emitStringWrapper(
MI,
MBB, SystemZ::MVST);
9633 case SystemZ::SRSTLoop:
9634 return emitStringWrapper(
MI,
MBB, SystemZ::SRST);
9635 case SystemZ::TBEGIN:
9636 return emitTransactionBegin(
MI,
MBB, SystemZ::TBEGIN,
false);
9637 case SystemZ::TBEGIN_nofloat:
9638 return emitTransactionBegin(
MI,
MBB, SystemZ::TBEGIN,
true);
9639 case SystemZ::TBEGINC:
9640 return emitTransactionBegin(
MI,
MBB, SystemZ::TBEGINC,
true);
9641 case SystemZ::LTEBRCompare_Pseudo:
9642 return emitLoadAndTestCmp0(
MI,
MBB, SystemZ::LTEBR);
9643 case SystemZ::LTDBRCompare_Pseudo:
9644 return emitLoadAndTestCmp0(
MI,
MBB, SystemZ::LTDBR);
9645 case SystemZ::LTXBRCompare_Pseudo:
9646 return emitLoadAndTestCmp0(
MI,
MBB, SystemZ::LTXBR);
9648 case SystemZ::PROBED_ALLOCA:
9649 return emitProbedAlloca(
MI,
MBB);
9651 case TargetOpcode::STACKMAP:
9652 case TargetOpcode::PATCHPOINT:
9663SystemZTargetLowering::getRepRegClassFor(
MVT VT)
const {
9664 if (VT == MVT::Untyped)
9665 return &SystemZ::ADDR128BitRegClass;
9691 DAG.
getMachineNode(SystemZ::EFPC, dl, {MVT::i32, MVT::Other}, Chain), 0);
9711 EVT VT =
Op.getValueType();
9712 Op =
Op.getOperand(0);
9713 EVT OpVT =
Op.getValueType();
9715 assert(OpVT.
isVector() &&
"Operand type for VECREDUCE_ADD is not a vector.");
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
amdgpu AMDGPU Register Bank Select
static bool isZeroVector(SDValue N)
Function Alias Analysis Results
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
static bool isUndef(ArrayRef< int > Mask)
unsigned const TargetRegisterInfo * TRI
uint64_t IntrinsicInst * II
const char LLVMTargetMachineRef TM
static bool isSelectPseudo(MachineInstr &MI)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static void adjustForLTGFR(Comparison &C)
static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0, SDValue Op1)
static bool isOnlyUsedByStores(SDValue StoredVal, SelectionDAG &DAG)
static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT, unsigned Opcode, SDValue Op0, SDValue Op1, SDValue &Even, SDValue &Odd)
static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Value)
static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In)
static bool isSimpleShift(SDValue N, unsigned &ShiftVal)
static bool isI128MovedToParts(LoadSDNode *LD, SDNode *&LoPart, SDNode *&HiPart)
static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1)
static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num)
static bool isVectorElementSwap(ArrayRef< int > M, EVT VT)
static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL, SDValue &AlignedAddr, SDValue &BitShift, SDValue &NegBitShift)
static bool isShlDoublePermute(const SmallVectorImpl< int > &Bytes, unsigned &StartIndex, unsigned &OpNo0, unsigned &OpNo1)
static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL, const Permute &P, SDValue Op0, SDValue Op1)
static SDNode * emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op, unsigned Opcode)
static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg)
static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode, unsigned &CCValid)
static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend, SDValue Op0, SDValue Op1, SDValue &Hi, SDValue &Lo)
static bool isF128MovedToParts(LoadSDNode *LD, SDNode *&LoPart, SDNode *&HiPart)
static void createPHIsForSelects(SmallVector< MachineInstr *, 8 > &Selects, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB, MachineBasicBlock *SinkMBB)
static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL, SDValue *Ops, const SmallVectorImpl< int > &Bytes)
static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, CmpMode Mode, bool &Invert)
static unsigned CCMaskForCondCode(ISD::CondCode CC)
static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static void adjustForFNeg(Comparison &C)
static bool isScalarToVector(SDValue Op)
static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg, unsigned CCValid, unsigned CCMask)
static bool matchPermute(const SmallVectorImpl< int > &Bytes, const Permute &P, unsigned &OpNo0, unsigned &OpNo1)
static bool isAddCarryChain(SDValue Carry)
static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static MachineOperand earlyUseOperand(MachineOperand Op)
static bool canUseSiblingCall(const CCState &ArgCCInfo, SmallVectorImpl< CCValAssign > &ArgLocs, SmallVectorImpl< ISD::OutputArg > &Outs)
static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask)
static bool getzOSCalleeAndADA(SelectionDAG &DAG, SDValue &Callee, SDValue &ADA, SDLoc &DL, SDValue &Chain)
static bool shouldSwapCmpOperands(const Comparison &C)
static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType)
static SDValue getADAEntry(SelectionDAG &DAG, SDValue Val, SDLoc DL, unsigned Offset, bool LoadAdr=false)
static SDNode * emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op, unsigned Opcode)
static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static bool getVPermMask(SDValue ShuffleOp, SmallVectorImpl< int > &Bytes)
static const Permute PermuteForms[]
static bool isI128MovedFromParts(SDValue Val, SDValue &LoPart, SDValue &HiPart)
static bool isSubBorrowChain(SDValue Carry)
static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts, unsigned OpNo)
static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op, bool IsNegative)
static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth, unsigned OpNo)
static SDValue expandBitCastI128ToF128(SelectionDAG &DAG, SDValue Src, const SDLoc &SL)
static SDValue tryBuildVectorShuffle(SelectionDAG &DAG, BuildVectorSDNode *BVN)
static unsigned getVectorComparison(ISD::CondCode CC, CmpMode Mode)
static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In)
static SDValue MergeInputChains(SDNode *N1, SDNode *N2)
static SDValue expandBitCastF128ToI128(SelectionDAG &DAG, SDValue Src, const SDLoc &SL)
static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask, uint64_t Mask, uint64_t CmpVal, unsigned ICmpType)
static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid)
static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL, SDValue Op, SDValue Chain)
static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, ISD::CondCode Cond, const SDLoc &DL, SDValue Chain=SDValue(), bool IsSignaling=false)
static bool checkCCKill(MachineInstr &MI, MachineBasicBlock *MBB)
static Register forceReg(MachineInstr &MI, MachineOperand &Base, const SystemZInstrInfo *TII)
static bool is32Bit(EVT VT)
static std::pair< unsigned, const TargetRegisterClass * > parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC, const unsigned *Map, unsigned Size)
static bool matchDoublePermute(const SmallVectorImpl< int > &Bytes, const Permute &P, SmallVectorImpl< int > &Transform)
static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode, SDValue Call, unsigned CCValid, uint64_t CC, ISD::CondCode Cond)
static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg)
static AddressingMode getLoadStoreAddrMode(bool HasVector, Type *Ty)
static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Op0, SDValue Op1)
static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth, unsigned OpNo)
static bool getShuffleInput(const SmallVectorImpl< int > &Bytes, unsigned Start, unsigned BytesPerElement, int &Base)
static AddressingMode supportedAddressingMode(Instruction *I, bool HasVector)
static bool isF128MovedFromParts(SDValue Val, SDValue &LoPart, SDValue &HiPart)
static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
Class for arbitrary precision integers.
APInt zext(unsigned width) const
Zero extend to a new width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool isSingleWord() const
Determine if this APInt just has one word to store value.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
an instruction that atomically reads a memory location, combines it with another value,...
BinOp getOperation() const
StringRef getValueAsString() const
Return the attribute's value as a string.
The address of a basic block.
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
CCState - This class holds information needed while lowering arguments and return values.
void AnalyzeCallResult(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeCallResult - Analyze the return values of a call, incorporating info about the passed values i...
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
uint64_t getFnAttributeAsParsedInteger(StringRef Kind, uint64_t Default=0) const
For a string attribute Kind, parse attribute as an integer.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
const GlobalObject * getAliaseeObject() const
bool hasPrivateLinkage() const
bool hasInternalLinkage() const
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
This class is used to represent ISD::LOAD nodes.
static auto integer_fixedlen_vector_valuetypes()
bool isVector() const
Return true if this is a vector value type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
static MVT getVectorVT(MVT VT, unsigned NumElements)
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void setAdjustsStack(bool V)
void setFrameAddressIsTaken(bool T)
void setReturnAddressIsTaken(bool s)
unsigned getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call.
void setMaxCallFrameSize(unsigned S)
MachineFunctionProperties & reset(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
reverse_iterator rbegin()
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Representation of each machine instruction.
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
bool killsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr kills the specified register.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Register getReg() const
getReg - Returns the register number.
bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
void setFlags(SDNodeFlags NewFlags)
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getMachineOpcode() const
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getGLOBAL_OFFSET_TABLE(EVT VT)
Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
constexpr size_t size() const
size - Get the string size.
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
A SystemZ-specific class detailing special use registers particular for calling conventions.
virtual int getStackPointerBias()=0
virtual int getReturnFunctionAddressRegister()=0
virtual int getCallFrameSize()=0
virtual int getStackPointerRegister()=0
A SystemZ-specific constant pool value.
static SystemZConstantPoolValue * Create(const GlobalValue *GV, SystemZCP::SystemZCPModifier Modifier)
unsigned getVarArgsFrameIndex() const
void setVarArgsFrameIndex(unsigned FI)
void setRegSaveFrameIndex(unsigned FI)
void incNumLocalDynamicTLSAccesses()
Register getVarArgsFirstGPR() const
void setADAVirtualRegister(Register Reg)
void setVarArgsFirstGPR(Register GPR)
Register getADAVirtualRegister() const
void setSizeOfFnParams(unsigned Size)
void setVarArgsFirstFPR(Register FPR)
unsigned getRegSaveFrameIndex() const
Register getVarArgsFirstFPR() const
const SystemZInstrInfo * getInstrInfo() const override
bool isPC32DBLSymbol(const GlobalValue *GV, CodeModel::Model CM) const
const TargetFrameLowering * getFrameLowering() const override
bool isTargetXPLINK64() const
SystemZCallingConventionRegisters * getSpecialRegisters() const
const SystemZRegisterInfo * getRegisterInfo() const override
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool hasInlineStackProbe(const MachineFunction &MF) const override
Returns true if stack probing through inline assembly is requested.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be cast by the IR-level AtomicExpand pass.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &, EVT) const override
Return the ValueType of the result of SETCC operations.
bool allowTruncateForTailCall(Type *, Type *) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool findOptimalMemOpLowering(std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes) const override
Determines the optimal series of memory ops to replace the memset / memcpy.
bool useSoftFloat() const override
std::pair< SDValue, SDValue > makeExternalCall(SDValue Chain, SelectionDAG &DAG, const char *CalleeName, EVT RetVT, ArrayRef< SDValue > Ops, CallingConv::ID CallConv, bool IsSigned, SDLoc DL, bool DoesNotReturn, bool IsReturnValueUsed) const
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
SystemZTargetLowering(const TargetMachine &TM, const SystemZSubtarget &STI)
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
TargetLowering::ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
Determine if the target supports unaligned memory accesses.
const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const override
Returns a 0 terminated array of registers that can be safely used as scratch registers.
TargetLowering::ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isTruncateFree(Type *, Type *) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine the number of bits in the operation that are sign bits.
void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const override
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be cast by the IR-level AtomicExpand pass into.
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
unsigned getStackProbeSize(const MachineFunction &MF) const
XPLINK64 calling convention specific use registers Particular to z/OS when in 64 bit mode.
int getCallFrameSize() final
Information about stack frame layout on the target.
unsigned getStackAlignment() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
virtual bool hasFP(const MachineFunction &MF) const =0
hasFP - Return true if the specified function should have a dedicated frame pointer register.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const
Returns true if arguments should be sign-extended in lib calls.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
void setAtomicLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Let target indicate that an extending atomic load of the specified type is legal.
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual const TargetRegisterClass * getRepRegClassFor(MVT VT) const
Return the 'representative' register class for the specified value type.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool findOptimalMemOpLowering(std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes) const
Determines the optimal series of memory ops to replace the memset / memcpy.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
unsigned getPointerSize(unsigned AS) const
Get the pointer size for this target.
CodeModel::Model getCodeModel() const
Returns the code model.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetInstrInfo * getInstrInfo() const
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
constexpr ScalarTy getFixedValue() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Define
Register definition.
@ System
Synchronized with respect to all concurrently executing threads.
@ MO_ADA_DATA_SYMBOL_ADDR
@ MO_ADA_DIRECT_FUNC_DESC
@ MO_ADA_INDIRECT_FUNC_DESC
const unsigned GR64Regs[16]
const unsigned VR128Regs[32]
const unsigned GR128Regs[16]
const unsigned FP32Regs[16]
const unsigned GR32Regs[16]
const unsigned FP64Regs[16]
const int64_t ELFCallFrameSize
const unsigned VR64Regs[32]
const unsigned FP128Regs[16]
const unsigned VR32Regs[32]
unsigned odd128(bool Is32bit)
const unsigned CCMASK_CMP_GE
static bool isImmHH(uint64_t Val)
const unsigned CCMASK_TEND
const unsigned CCMASK_CS_EQ
const unsigned CCMASK_TBEGIN
const MCPhysReg ELFArgFPRs[ELFNumArgFPRs]
MachineBasicBlock * splitBlockBefore(MachineBasicBlock::iterator MI, MachineBasicBlock *MBB)
const unsigned CCMASK_TM_SOME_1
const unsigned CCMASK_LOGICAL_CARRY
const unsigned TDCMASK_NORMAL_MINUS
const unsigned CCMASK_TDC
const unsigned CCMASK_FCMP
const unsigned CCMASK_TM_SOME_0
static bool isImmHL(uint64_t Val)
const unsigned TDCMASK_SUBNORMAL_MINUS
const unsigned TDCMASK_NORMAL_PLUS
const unsigned CCMASK_CMP_GT
const unsigned TDCMASK_QNAN_MINUS
const unsigned CCMASK_ANY
const unsigned CCMASK_ARITH
const unsigned CCMASK_TM_MIXED_MSB_0
const unsigned TDCMASK_SUBNORMAL_PLUS
static bool isImmLL(uint64_t Val)
const unsigned VectorBits
static bool isImmLH(uint64_t Val)
MachineBasicBlock * emitBlockAfter(MachineBasicBlock *MBB)
const unsigned TDCMASK_INFINITY_PLUS
unsigned reverseCCMask(unsigned CCMask)
const unsigned CCMASK_TM_ALL_0
const unsigned CCMASK_CMP_LE
const unsigned CCMASK_CMP_O
const unsigned CCMASK_CMP_EQ
const unsigned VectorBytes
const unsigned TDCMASK_INFINITY_MINUS
const unsigned CCMASK_ICMP
const unsigned CCMASK_VCMP_ALL
MachineBasicBlock * splitBlockAfter(MachineBasicBlock::iterator MI, MachineBasicBlock *MBB)
const unsigned CCMASK_VCMP
const unsigned CCMASK_TM_MIXED_MSB_1
const unsigned CCMASK_TM_MSB_0
const unsigned CCMASK_ARITH_OVERFLOW
const unsigned CCMASK_CS_NE
const unsigned TDCMASK_SNAN_PLUS
const unsigned CCMASK_CMP_LT
const unsigned CCMASK_CMP_NE
const unsigned TDCMASK_ZERO_PLUS
const unsigned TDCMASK_QNAN_PLUS
const unsigned TDCMASK_ZERO_MINUS
unsigned even128(bool Is32bit)
const unsigned CCMASK_TM_ALL_1
const unsigned CCMASK_LOGICAL_BORROW
const unsigned ELFNumArgFPRs
const unsigned CCMASK_CMP_UO
const unsigned CCMASK_LOGICAL
const unsigned CCMASK_TM_MSB_1
const unsigned TDCMASK_SNAN_MINUS
Reg
All possible values of the reg field in the ModR/M byte.
support::ulittle32_t Word
NodeAddr< CodeNode * > Code
constexpr const char32_t SBase
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
testing::Matcher< const detail::ErrorHolder & > Failed()
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void dumpBytes(ArrayRef< uint8_t > Bytes, raw_ostream &OS)
Convert ‘Bytes’ to a hex string and output to ‘OS’.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Mul
Product of integers.
DWARFExpression::Operation Op
constexpr unsigned BitWidth
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
AddressingMode(bool LongDispl, bool IdxReg)
This struct is a compact representation of a valid (non-zero power of two) alignment.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isRound() const
Return true if the size is a power-of-two number of bytes.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
KnownBits anyextOrTrunc(unsigned BitWidth) const
Return known bits for an "any" extension or truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
SystemZVectorConstantInfo(APInt IntImm)
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
CallLoweringInfo & setNoReturn(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})