78#include "llvm/IR/IntrinsicsAArch64.h"
79#include "llvm/IR/IntrinsicsWebAssembly.h"
109using namespace PatternMatch;
110using namespace SwitchCG;
112#define DEBUG_TYPE "isel"
120 cl::desc(
"Insert the experimental `assertalign` node."),
125 cl::desc(
"Generate low-precision inline sequences "
126 "for some float libcalls"),
132 cl::desc(
"Set the case probability threshold for peeling the case from a "
133 "switch statement. A value greater than 100 will void this "
153 const SDValue *Parts,
unsigned NumParts,
155 std::optional<CallingConv::ID>
CC);
164 unsigned NumParts,
MVT PartVT,
EVT ValueVT,
const Value *V,
165 std::optional<CallingConv::ID>
CC = std::nullopt,
166 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
170 PartVT, ValueVT,
CC))
177 assert(NumParts > 0 &&
"No parts to assemble!");
188 unsigned RoundBits = PartBits * RoundParts;
189 EVT RoundVT = RoundBits == ValueBits ?
195 if (RoundParts > 2) {
199 RoundParts / 2, PartVT, HalfVT, V);
210 if (RoundParts < NumParts) {
212 unsigned OddParts = NumParts - RoundParts;
232 assert(ValueVT ==
EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
243 !PartVT.
isVector() &&
"Unexpected split");
254 if (PartEVT == ValueVT)
258 ValueVT.
bitsLT(PartEVT)) {
271 if (ValueVT.
bitsLT(PartEVT)) {
276 Val = DAG.
getNode(*AssertOp,
DL, PartEVT, Val,
295 if (PartEVT == MVT::x86mmx && ValueVT.
isInteger() &&
296 ValueVT.
bitsLT(PartEVT)) {
305 const Twine &ErrMsg) {
306 const Instruction *
I = dyn_cast_or_null<Instruction>(V);
310 const char *AsmError =
", possible invalid constraint for vector type";
311 if (
const CallInst *CI = dyn_cast<CallInst>(
I))
312 if (CI->isInlineAsm())
324 const SDValue *Parts,
unsigned NumParts,
326 std::optional<CallingConv::ID> CallConv) {
328 assert(NumParts > 0 &&
"No parts to assemble!");
329 const bool IsABIRegCopy = CallConv.has_value();
338 unsigned NumIntermediates;
343 *DAG.
getContext(), *CallConv, ValueVT, IntermediateVT,
344 NumIntermediates, RegisterVT);
348 NumIntermediates, RegisterVT);
351 assert(NumRegs == NumParts &&
"Part count doesn't match vector breakdown!");
353 assert(RegisterVT == PartVT &&
"Part type doesn't match vector breakdown!");
356 "Part type sizes don't match!");
360 if (NumIntermediates == NumParts) {
363 for (
unsigned i = 0; i != NumParts; ++i)
365 PartVT, IntermediateVT, V, CallConv);
366 }
else if (NumParts > 0) {
369 assert(NumParts % NumIntermediates == 0 &&
370 "Must expand into a divisible number of parts!");
371 unsigned Factor = NumParts / NumIntermediates;
372 for (
unsigned i = 0; i != NumIntermediates; ++i)
374 PartVT, IntermediateVT, V, CallConv);
389 DL, BuiltVectorTy, Ops);
395 if (PartEVT == ValueVT)
411 "Cannot narrow, it would be a lossy transformation");
417 if (PartEVT == ValueVT)
442 }
else if (ValueVT.
bitsLT(PartEVT)) {
451 *DAG.
getContext(), V,
"non-trivial scalar-to-vector conversion");
482 std::optional<CallingConv::ID> CallConv);
489 unsigned NumParts,
MVT PartVT,
const Value *V,
490 std::optional<CallingConv::ID> CallConv = std::nullopt,
504 unsigned OrigNumParts = NumParts;
506 "Copying to an illegal type!");
512 EVT PartEVT = PartVT;
513 if (PartEVT == ValueVT) {
514 assert(NumParts == 1 &&
"No-op copy with multiple parts!");
523 assert(NumParts == 1 &&
"Do not know what to promote to!");
534 "Unknown mismatch!");
536 Val = DAG.
getNode(ExtendKind,
DL, ValueVT, Val);
537 if (PartVT == MVT::x86mmx)
542 assert(NumParts == 1 && PartEVT != ValueVT);
548 "Unknown mismatch!");
551 if (PartVT == MVT::x86mmx)
558 "Failed to tile the value with PartVT!");
561 if (PartEVT != ValueVT) {
563 "scalar-to-vector conversion failed");
572 if (NumParts & (NumParts - 1)) {
575 "Do not know what to expand to!");
577 unsigned RoundBits = RoundParts * PartBits;
578 unsigned OddParts = NumParts - RoundParts;
587 std::reverse(Parts + RoundParts, Parts + NumParts);
589 NumParts = RoundParts;
601 for (
unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
602 for (
unsigned i = 0; i < NumParts; i += StepSize) {
603 unsigned ThisBits = StepSize * PartBits / 2;
606 SDValue &Part1 = Parts[i+StepSize/2];
613 if (ThisBits == PartBits && ThisVT != PartVT) {
621 std::reverse(Parts, Parts + OrigNumParts);
638 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
643 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
645 "Cannot widen to illegal type");
648 }
else if (PartEVT != ValueEVT) {
663 Ops.
append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
674 std::optional<CallingConv::ID> CallConv) {
678 const bool IsABIRegCopy = CallConv.has_value();
681 EVT PartEVT = PartVT;
682 if (PartEVT == ValueVT) {
701 TargetLowering::TypeWidenVector) {
719 "lossy conversion of vector to scalar type");
734 unsigned NumIntermediates;
738 *DAG.
getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
743 NumIntermediates, RegisterVT);
746 assert(NumRegs == NumParts &&
"Part count doesn't match vector breakdown!");
748 assert(RegisterVT == PartVT &&
"Part type doesn't match vector breakdown!");
751 "Mixing scalable and fixed vectors when copying in parts");
753 std::optional<ElementCount> DestEltCnt;
763 if (ValueVT == BuiltVectorTy) {
787 for (
unsigned i = 0; i != NumIntermediates; ++i) {
802 if (NumParts == NumIntermediates) {
805 for (
unsigned i = 0; i != NumParts; ++i)
807 }
else if (NumParts > 0) {
810 assert(NumIntermediates != 0 &&
"division by zero");
811 assert(NumParts % NumIntermediates == 0 &&
812 "Must expand into a divisible number of parts!");
813 unsigned Factor = NumParts / NumIntermediates;
814 for (
unsigned i = 0; i != NumIntermediates; ++i)
821 EVT valuevt, std::optional<CallingConv::ID>
CC)
822 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
823 RegCount(1, regs.
size()), CallConv(
CC) {}
827 std::optional<CallingConv::ID>
CC) {
841 for (
unsigned i = 0; i != NumRegs; ++i)
843 RegVTs.push_back(RegisterVT);
872 for (
unsigned i = 0; i != NumRegs; ++i) {
878 *Glue =
P.getValue(2);
881 Chain =
P.getValue(1);
910 EVT FromVT(MVT::Other);
914 }
else if (NumSignBits > 1) {
922 assert(FromVT != MVT::Other);
958 NumParts, RegisterVT, V,
CallConv, ExtendKind);
964 for (
unsigned i = 0; i != NumRegs; ++i) {
976 if (NumRegs == 1 || Glue)
987 Chain = Chains[NumRegs-1];
993 unsigned MatchingIdx,
const SDLoc &dl,
995 std::vector<SDValue> &Ops)
const {
1000 Flag.setMatchingOp(MatchingIdx);
1009 Flag.setRegClass(RC->
getID());
1020 "No 1:1 mapping from clobbers to regs?");
1028 "If we clobbered the stack pointer, MFI should know about it.");
1037 for (
unsigned i = 0; i != NumRegs; ++i) {
1039 unsigned TheReg =
Regs[Reg++];
1050 unsigned RegCount = std::get<0>(CountAndVT);
1051 MVT RegisterVT = std::get<1>(CountAndVT);
1075 UnusedArgNodeMap.clear();
1077 PendingExports.clear();
1078 PendingConstrainedFP.clear();
1079 PendingConstrainedFPStrict.clear();
1087 DanglingDebugInfoMap.clear();
1094 if (Pending.
empty())
1100 unsigned i = 0, e = Pending.
size();
1101 for (; i != e; ++i) {
1102 assert(Pending[i].getNode()->getNumOperands() > 1);
1103 if (Pending[i].getNode()->getOperand(0) == Root)
1111 if (Pending.
size() == 1)
1130 PendingConstrainedFP.size() +
1131 PendingConstrainedFPStrict.size());
1133 PendingConstrainedFP.end());
1134 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1135 PendingConstrainedFPStrict.end());
1136 PendingConstrainedFP.clear();
1137 PendingConstrainedFPStrict.clear();
1144 PendingExports.append(PendingConstrainedFPStrict.begin(),
1145 PendingConstrainedFPStrict.end());
1146 PendingConstrainedFPStrict.clear();
1147 return updateRoot(PendingExports);
1152 if (
I.isTerminator()) {
1153 HandlePHINodesInSuccessorBlocks(
I.getParent());
1161 for (
auto It = FnVarLocs->locs_begin(&
I),
End = FnVarLocs->locs_end(&
I);
1163 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1165 if (It->Values.isKillLocation(It->Expr)) {
1171 It->Values.hasArgList()))
1177 if (!isa<DbgInfoIntrinsic>(
I))
1183 bool NodeInserted =
false;
1184 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1185 MDNode *PCSectionsMD =
I.getMetadata(LLVMContext::MD_pcsections);
1187 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1188 DAG, [&](
SDNode *) { NodeInserted =
true; });
1194 !isa<GCStatepointInst>(
I))
1199 auto It = NodeMap.find(&
I);
1200 if (It != NodeMap.end()) {
1202 }
else if (NodeInserted) {
1205 errs() <<
"warning: loosing !pcsections metadata ["
1206 <<
I.getModule()->getName() <<
"]\n";
1215void SelectionDAGBuilder::visitPHI(
const PHINode &) {
1225#define HANDLE_INST(NUM, OPCODE, CLASS) \
1226 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1227#include "llvm/IR/Instruction.def"
1276 "DbgValueInst without an ArgList should have a single location "
1278 DanglingDebugInfoMap[DI->
getValue(0)].emplace_back(DI, Order);
1284 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1287 if (DanglingVariable == Variable && Expr->
fragmentsOverlap(DanglingExpr)) {
1288 LLVM_DEBUG(
dbgs() <<
"Dropping dangling debug info for " << printDDI(DDI)
1295 for (
auto &DDIMI : DanglingDebugInfoMap) {
1296 DanglingDebugInfoVector &DDIV = DDIMI.second;
1300 for (
auto &DDI : DDIV)
1301 if (isMatchingDbgValue(DDI))
1304 erase_if(DDIV, isMatchingDbgValue);
1312 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1313 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1316 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1317 for (
auto &DDI : DDIV) {
1320 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1324 "Expected inlined-at fields to agree");
1333 if (!EmitFuncArgumentDbgValue(V, Variable, Expr,
DL,
1334 FuncArgumentDbgValueKind::Value, Val)) {
1335 LLVM_DEBUG(
dbgs() <<
"Resolve dangling debug info for " << printDDI(DDI)
1343 <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to "
1344 << ValSDNodeOrder <<
"\n");
1345 SDV = getDbgValue(Val, Variable, Expr,
DL,
1346 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1350 << printDDI(DDI) <<
" in EmitFuncArgumentDbgValue\n");
1352 LLVM_DEBUG(
dbgs() <<
"Dropping debug info for " << printDDI(DDI) <<
"\n");
1367 Value *V = DDI.getVariableLocationOp(0);
1372 unsigned SDOrder = DDI.getSDNodeOrder();
1376 bool StackValue =
true;
1385 while (isa<Instruction>(V)) {
1400 if (!AdditionalValues.
empty())
1410 dbgs() <<
"Salvaged debug location info for:\n " << *Var <<
"\n"
1411 << *OrigV <<
"\nBy stripping back to:\n " << *V <<
"\n");
1419 assert(OrigV &&
"V shouldn't be null");
1423 LLVM_DEBUG(
dbgs() <<
"Dropping debug value info for:\n " << printDDI(DDI)
1441 unsigned Order,
bool IsVariadic) {
1446 for (
const Value *V : Values) {
1448 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1449 isa<ConstantPointerNull>(V)) {
1455 if (
auto *CE = dyn_cast<ConstantExpr>(V))
1456 if (CE->getOpcode() == Instruction::IntToPtr) {
1463 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1474 if (!
N.getNode() && isa<Argument>(V))
1475 N = UnusedArgNodeMap[V];
1479 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1480 FuncArgumentDbgValueKind::Value,
N))
1482 if (
auto *FISDN = dyn_cast<FrameIndexSDNode>(
N.getNode())) {
1507 bool IsParamOfFunc =
1517 unsigned Reg = VMI->second;
1521 V->getType(), std::nullopt);
1527 unsigned BitsToDescribe = 0;
1529 BitsToDescribe = *VarSize;
1531 BitsToDescribe = Fragment->SizeInBits;
1534 if (
Offset >= BitsToDescribe)
1537 unsigned RegisterSize = RegAndSize.second;
1538 unsigned FragmentSize = (
Offset + RegisterSize > BitsToDescribe)
1539 ? BitsToDescribe -
Offset
1542 Expr,
Offset, FragmentSize);
1546 Var, *FragmentExpr, RegAndSize.first,
false, DbgLoc, SDNodeOrder);
1565 SDNodeOrder, IsVariadic);
1572 for (
auto &Pair : DanglingDebugInfoMap)
1573 for (
auto &DDI : Pair.second)
1605 if (
N.getNode())
return N;
1647 if (
const Constant *
C = dyn_cast<Constant>(V)) {
1656 if (isa<ConstantPointerNull>(
C)) {
1657 unsigned AS = V->getType()->getPointerAddressSpace();
1665 if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(
C))
1668 if (isa<UndefValue>(
C) && !V->getType()->isAggregateType())
1672 visit(CE->getOpcode(), *CE);
1674 assert(N1.
getNode() &&
"visit didn't populate the NodeMap!");
1678 if (isa<ConstantStruct>(
C) || isa<ConstantArray>(
C)) {
1680 for (
const Use &U :
C->operands()) {
1686 for (
unsigned i = 0, e = Val->
getNumValues(); i != e; ++i)
1694 dyn_cast<ConstantDataSequential>(
C)) {
1696 for (
unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1700 for (
unsigned i = 0, e = Val->
getNumValues(); i != e; ++i)
1704 if (isa<ArrayType>(CDS->getType()))
1709 if (
C->getType()->isStructTy() ||
C->getType()->isArrayTy()) {
1710 assert((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(
C)) &&
1711 "Unknown struct or array constant!");
1715 unsigned NumElts = ValueVTs.
size();
1719 for (
unsigned i = 0; i != NumElts; ++i) {
1720 EVT EltVT = ValueVTs[i];
1721 if (isa<UndefValue>(
C))
1735 if (
const auto *Equiv = dyn_cast<DSOLocalEquivalent>(
C))
1736 return getValue(Equiv->getGlobalValue());
1738 if (
const auto *
NC = dyn_cast<NoCFIValue>(
C))
1741 VectorType *VecTy = cast<VectorType>(V->getType());
1747 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1748 for (
unsigned i = 0; i != NumElements; ++i)
1754 if (isa<ConstantAggregateZero>(
C)) {
1772 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1781 if (
const Instruction *Inst = dyn_cast<Instruction>(V)) {
1785 Inst->getType(), std::nullopt);
1793 if (
const auto *BB = dyn_cast<BasicBlock>(V))
1799void SelectionDAGBuilder::visitCatchPad(
const CatchPadInst &
I) {
1808 if (IsMSVCCXX || IsCoreCLR)
1835 Value *ParentPad =
I.getCatchSwitchParentPad();
1837 if (isa<ConstantTokenNone>(ParentPad))
1840 SuccessorColor = cast<Instruction>(ParentPad)->
getParent();
1841 assert(SuccessorColor &&
"No parent funclet for catchret!");
1843 assert(SuccessorColorMBB &&
"No MBB for SuccessorColor!");
1852void SelectionDAGBuilder::visitCleanupPad(
const CleanupPadInst &CPI) {
1896 if (isa<CleanupPadInst>(Pad)) {
1898 UnwindDests.emplace_back(FuncInfo.
MBBMap[EHPadBB], Prob);
1899 UnwindDests.back().first->setIsEHScopeEntry();
1901 }
else if (
const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1904 for (
const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1905 UnwindDests.emplace_back(FuncInfo.
MBBMap[CatchPadBB], Prob);
1906 UnwindDests.back().first->setIsEHScopeEntry();
1937 assert(UnwindDests.size() <= 1 &&
1938 "There should be at most one unwind destination for wasm");
1945 if (isa<LandingPadInst>(Pad)) {
1947 UnwindDests.emplace_back(FuncInfo.
MBBMap[EHPadBB], Prob);
1949 }
else if (isa<CleanupPadInst>(Pad)) {
1952 UnwindDests.emplace_back(FuncInfo.
MBBMap[EHPadBB], Prob);
1953 UnwindDests.
back().first->setIsEHScopeEntry();
1954 UnwindDests.back().first->setIsEHFuncletEntry();
1956 }
else if (
const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1958 for (
const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1959 UnwindDests.emplace_back(FuncInfo.
MBBMap[CatchPadBB], Prob);
1961 if (IsMSVCCXX || IsCoreCLR)
1962 UnwindDests.back().first->setIsEHFuncletEntry();
1964 UnwindDests.back().first->setIsEHScopeEntry();
1966 NewEHPadBB = CatchSwitch->getUnwindDest();
1972 if (BPI && NewEHPadBB)
1974 EHPadBB = NewEHPadBB;
1981 auto UnwindDest =
I.getUnwindDest();
1988 for (
auto &UnwindDest : UnwindDests) {
1989 UnwindDest.first->setIsEHPad();
1990 addSuccessorWithProb(
FuncInfo.
MBB, UnwindDest.first, UnwindDest.second);
2000void SelectionDAGBuilder::visitCatchSwitch(
const CatchSwitchInst &CSI) {
2004void SelectionDAGBuilder::visitRet(
const ReturnInst &
I) {
2018 if (
I.getParent()->getTerminatingDeoptimizeCall()) {
2025 const Function *
F =
I.getParent()->getParent();
2044 unsigned NumValues = ValueVTs.
size();
2047 Align BaseAlign =
DL.getPrefTypeAlign(
I.getOperand(0)->getType());
2048 for (
unsigned i = 0; i != NumValues; ++i) {
2055 if (MemVTs[i] != ValueVTs[i])
2065 MVT::Other, Chains);
2066 }
else if (
I.getNumOperands() != 0) {
2069 unsigned NumValues = ValueVTs.
size();
2073 const Function *
F =
I.getParent()->getParent();
2076 I.getOperand(0)->getType(),
F->getCallingConv(),
2080 if (
F->getAttributes().hasRetAttr(Attribute::SExt))
2082 else if (
F->getAttributes().hasRetAttr(Attribute::ZExt))
2086 bool RetInReg =
F->getAttributes().hasRetAttr(Attribute::InReg);
2088 for (
unsigned j = 0;
j != NumValues; ++
j) {
2089 EVT VT = ValueVTs[
j];
2101 &Parts[0], NumParts, PartVT, &
I,
CC, ExtendKind);
2108 if (
I.getOperand(0)->getType()->isPointerTy()) {
2110 Flags.setPointerAddrSpace(
2111 cast<PointerType>(
I.getOperand(0)->getType())->getAddressSpace());
2114 if (NeedsRegBlock) {
2115 Flags.setInConsecutiveRegs();
2116 if (j == NumValues - 1)
2117 Flags.setInConsecutiveRegsLast();
2126 for (
unsigned i = 0; i < NumParts; ++i) {
2128 Parts[i].getValueType().getSimpleVT(),
2139 const Function *
F =
I.getParent()->getParent();
2141 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2144 Flags.setSwiftError();
2163 "LowerReturn didn't return a valid chain!");
2174 if (V->getType()->isEmptyTy())
2179 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2180 "Unused value assigned virtual registers!");
2190 if (!isa<Instruction>(V) && !isa<Argument>(V))
return;
2203 if (
const Instruction *VI = dyn_cast<Instruction>(V)) {
2205 if (VI->getParent() == FromBB)
2214 if (isa<Argument>(V)) {
2231 const BasicBlock *SrcBB = Src->getBasicBlock();
2232 const BasicBlock *DstBB = Dst->getBasicBlock();
2236 auto SuccSize = std::max<uint32_t>(
succ_size(SrcBB), 1);
2246 Src->addSuccessorWithoutProb(Dst);
2249 Prob = getEdgeProbability(Src, Dst);
2250 Src->addSuccessor(Dst, Prob);
2256 return I->getParent() == BB;
2276 if (
const CmpInst *BOp = dyn_cast<CmpInst>(
Cond)) {
2280 if (CurBB == SwitchBB ||
2286 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2291 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2297 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1),
nullptr,
2299 SL->SwitchCases.push_back(CB);
2308 SL->SwitchCases.push_back(CB);
2331 const Value *BOpOp0, *BOpOp1;
2345 if (BOpc == Instruction::And)
2346 BOpc = Instruction::Or;
2347 else if (BOpc == Instruction::Or)
2348 BOpc = Instruction::And;
2354 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->
hasOneUse();
2359 TProb, FProb, InvertCond);
2369 if (Opc == Instruction::Or) {
2390 auto NewTrueProb = TProb / 2;
2391 auto NewFalseProb = TProb / 2 + FProb;
2394 NewFalseProb, InvertCond);
2401 Probs[1], InvertCond);
2403 assert(Opc == Instruction::And &&
"Unknown merge op!");
2423 auto NewTrueProb = TProb + FProb / 2;
2424 auto NewFalseProb = FProb / 2;
2427 NewFalseProb, InvertCond);
2434 Probs[1], InvertCond);
2443 if (Cases.size() != 2)
return true;
2447 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2448 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2449 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2450 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2456 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2457 Cases[0].
CC == Cases[1].
CC &&
2458 isa<Constant>(Cases[0].CmpRHS) &&
2459 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2460 if (Cases[0].
CC ==
ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2462 if (Cases[0].
CC ==
ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2469void SelectionDAGBuilder::visitBr(
const BranchInst &
I) {
2475 if (
I.isUnconditional()) {
2481 if (Succ0MBB != NextBlock(BrMBB) ||
2494 const Value *CondVal =
I.getCondition();
2514 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2516 BOp->
hasOneUse() && !
I.hasMetadata(LLVMContext::MD_unpredictable)) {
2518 const Value *BOp0, *BOp1;
2521 Opcode = Instruction::And;
2523 Opcode = Instruction::Or;
2528 getEdgeProbability(BrMBB, Succ0MBB),
2529 getEdgeProbability(BrMBB, Succ1MBB),
2534 assert(
SL->SwitchCases[0].ThisBB == BrMBB &&
"Unexpected lowering!");
2538 for (
unsigned i = 1, e =
SL->SwitchCases.size(); i != e; ++i) {
2545 SL->SwitchCases.erase(
SL->SwitchCases.begin());
2551 for (
unsigned i = 1, e =
SL->SwitchCases.size(); i != e; ++i)
2554 SL->SwitchCases.clear();
2560 nullptr, Succ0MBB, Succ1MBB, BrMBB,
getCurSDLoc());
2579 if (CB.
TrueBB != NextBlock(SwitchBB)) {
2621 if (cast<ConstantInt>(CB.
CmpLHS)->isMinValue(
true)) {
2642 if (CB.
TrueBB == NextBlock(SwitchBB)) {
2666 assert(JT.Reg != -1U &&
"Should lower JT Header first!");
2672 MVT::Other,
Index.getValue(1),
2698 unsigned JumpTableReg =
2701 JumpTableReg, SwitchOp);
2702 JT.Reg = JumpTableReg;
2714 MVT::Other, CopyTo, CMP,
2718 if (JT.MBB != NextBlock(SwitchBB))
2725 if (JT.MBB != NextBlock(SwitchBB))
2752 if (PtrTy != PtrMemTy)
2800 Entry.Node = GuardVal;
2802 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
2803 Entry.IsInReg =
true;
2804 Args.push_back(Entry);
2810 getValue(GuardCheckFn), std::move(Args));
2812 std::pair<SDValue, SDValue> Result = TLI.
LowerCallTo(CLI);
2826 Guard =
DAG.
getLoad(PtrMemTy, dl, Chain, GuardPtr,
2863 TLI.
makeLibCall(
DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2894 bool UsePtrType =
false;
2898 for (
unsigned i = 0, e =
B.Cases.size(); i != e; ++i)
2918 if (!
B.FallthroughUnreachable)
2919 addSuccessorWithProb(SwitchBB,
B.Default,
B.DefaultProb);
2920 addSuccessorWithProb(SwitchBB,
MBB,
B.Prob);
2924 if (!
B.FallthroughUnreachable) {
2937 if (
MBB != NextBlock(SwitchBB))
2956 if (PopCount == 1) {
2963 }
else if (PopCount == BB.
Range) {
2982 addSuccessorWithProb(SwitchBB,
B.TargetBB,
B.ExtraProb);
2984 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2995 if (NextMBB != NextBlock(SwitchBB))
3002void SelectionDAGBuilder::visitInvoke(
const InvokeInst &
I) {
3013 assert(!
I.hasOperandBundlesOtherThan(
3014 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3015 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3016 LLVMContext::OB_cfguardtarget,
3017 LLVMContext::OB_clang_arc_attachedcall}) &&
3018 "Cannot lower invokes with arbitrary operand bundles yet!");
3020 const Value *Callee(
I.getCalledOperand());
3021 const Function *Fn = dyn_cast<Function>(Callee);
3022 if (isa<InlineAsm>(Callee))
3023 visitInlineAsm(
I, EHPadBB);
3028 case Intrinsic::donothing:
3030 case Intrinsic::seh_try_begin:
3031 case Intrinsic::seh_scope_begin:
3032 case Intrinsic::seh_try_end:
3033 case Intrinsic::seh_scope_end:
3039 case Intrinsic::experimental_patchpoint_void:
3040 case Intrinsic::experimental_patchpoint_i64:
3041 visitPatchpoint(
I, EHPadBB);
3043 case Intrinsic::experimental_gc_statepoint:
3046 case Intrinsic::wasm_rethrow: {
3075 if (!isa<GCStatepointInst>(
I)) {
3087 addSuccessorWithProb(InvokeMBB, Return);
3088 for (
auto &UnwindDest : UnwindDests) {
3089 UnwindDest.first->setIsEHPad();
3090 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3099void SelectionDAGBuilder::visitCallBr(
const CallBrInst &
I) {
3104 assert(!
I.hasOperandBundlesOtherThan(
3105 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3106 "Cannot lower callbrs with arbitrary operand bundles yet!");
3108 assert(
I.isInlineAsm() &&
"Only know how to handle inlineasm callbr");
3114 Dests.
insert(
I.getDefaultDest());
3119 for (
unsigned i = 0, e =
I.getNumIndirectDests(); i < e; ++i) {
3122 Target->setIsInlineAsmBrIndirectTarget();
3123 Target->setMachineBlockAddressTaken();
3124 Target->setLabelMustBeEmitted();
3126 if (Dests.
insert(Dest).second)
3137void SelectionDAGBuilder::visitResume(
const ResumeInst &RI) {
3138 llvm_unreachable(
"SelectionDAGBuilder shouldn't visit resume instructions!");
3141void SelectionDAGBuilder::visitLandingPad(
const LandingPadInst &LP) {
3143 "Call to landingpad not in landing pad!");
3163 assert(ValueVTs.
size() == 2 &&
"Only two-valued landingpads are supported");
3193 if (JTB.first.HeaderBB ==
First)
3194 JTB.first.HeaderBB =
Last;
3207 for (
unsigned i = 0, e =
I.getNumSuccessors(); i != e; ++i) {
3209 bool Inserted =
Done.insert(BB).second;
3214 addSuccessorWithProb(IndirectBrMBB, Succ);
3233 if (
const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3234 if (
Call->doesNotReturn())
3243void SelectionDAGBuilder::visitUnary(
const User &
I,
unsigned Opcode) {
3245 if (
auto *FPOp = dyn_cast<FPMathOperator>(&
I))
3246 Flags.copyFMF(*FPOp);
3254void SelectionDAGBuilder::visitBinary(
const User &
I,
unsigned Opcode) {
3256 if (
auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&
I)) {
3257 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3258 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3260 if (
auto *ExactOp = dyn_cast<PossiblyExactOperator>(&
I))
3261 Flags.setExact(ExactOp->isExact());
3262 if (
auto *FPOp = dyn_cast<FPMathOperator>(&
I))
3263 Flags.copyFMF(*FPOp);
3272void SelectionDAGBuilder::visitShift(
const User &
I,
unsigned Opcode) {
3281 if (!
I.getType()->isVectorTy() && Op2.
getValueType() != ShiftTy) {
3283 "Unexpected shift type");
3294 dyn_cast<const OverflowingBinaryOperator>(&
I)) {
3295 nuw = OFBinOp->hasNoUnsignedWrap();
3296 nsw = OFBinOp->hasNoSignedWrap();
3299 dyn_cast<const PossiblyExactOperator>(&
I))
3300 exact = ExactOp->isExact();
3303 Flags.setExact(exact);
3304 Flags.setNoSignedWrap(nsw);
3305 Flags.setNoUnsignedWrap(nuw);
3311void SelectionDAGBuilder::visitSDiv(
const User &
I) {
3316 Flags.setExact(isa<PossiblyExactOperator>(&
I) &&
3317 cast<PossiblyExactOperator>(&
I)->isExact());
3322void SelectionDAGBuilder::visitICmp(
const User &
I) {
3324 if (
const ICmpInst *IC = dyn_cast<ICmpInst>(&
I))
3325 predicate = IC->getPredicate();
3326 else if (
const ConstantExpr *IC = dyn_cast<ConstantExpr>(&
I))
3349void SelectionDAGBuilder::visitFCmp(
const User &
I) {
3351 if (
const FCmpInst *FC = dyn_cast<FCmpInst>(&
I))
3352 predicate =
FC->getPredicate();
3353 else if (
const ConstantExpr *FC = dyn_cast<ConstantExpr>(&
I))
3359 auto *FPMO = cast<FPMathOperator>(&
I);
3364 Flags.copyFMF(*FPMO);
3376 return isa<SelectInst>(V);
3380void SelectionDAGBuilder::visitSelect(
const User &
I) {
3384 unsigned NumValues = ValueVTs.
size();
3385 if (NumValues == 0)
return;
3395 bool IsUnaryAbs =
false;
3396 bool Negate =
false;
3399 if (
auto *FPOp = dyn_cast<FPMathOperator>(&
I))
3400 Flags.copyFMF(*FPOp);
3402 Flags.setUnpredictable(
3403 cast<SelectInst>(
I).getMetadata(LLVMContext::MD_unpredictable));
3407 EVT VT = ValueVTs[0];
3419 bool UseScalarMinMax = VT.
isVector() &&
3428 switch (SPR.Flavor) {
3434 switch (SPR.NaNBehavior) {
3447 switch (SPR.NaNBehavior) {
3491 for (
unsigned i = 0; i != NumValues; ++i) {
3500 for (
unsigned i = 0; i != NumValues; ++i) {
3514void SelectionDAGBuilder::visitTrunc(
const User &
I) {
3522void SelectionDAGBuilder::visitZExt(
const User &
I) {
3531void SelectionDAGBuilder::visitSExt(
const User &
I) {
3540void SelectionDAGBuilder::visitFPTrunc(
const User &
I) {
3551void SelectionDAGBuilder::visitFPExt(
const User &
I) {
3559void SelectionDAGBuilder::visitFPToUI(
const User &
I) {
3567void SelectionDAGBuilder::visitFPToSI(
const User &
I) {
3575void SelectionDAGBuilder::visitUIToFP(
const User &
I) {
3583void SelectionDAGBuilder::visitSIToFP(
const User &
I) {
3591void SelectionDAGBuilder::visitPtrToInt(
const User &
I) {
3605void SelectionDAGBuilder::visitIntToPtr(
const User &
I) {
3617void SelectionDAGBuilder::visitBitCast(
const User &
I) {
3625 if (DestVT !=
N.getValueType())
3632 else if(
ConstantInt *
C = dyn_cast<ConstantInt>(
I.getOperand(0)))
3639void SelectionDAGBuilder::visitAddrSpaceCast(
const User &
I) {
3641 const Value *SV =
I.getOperand(0);
3646 unsigned DestAS =
I.getType()->getPointerAddressSpace();
3654void SelectionDAGBuilder::visitInsertElement(
const User &
I) {
3662 InVec, InVal, InIdx));
3665void SelectionDAGBuilder::visitExtractElement(
const User &
I) {
3675void SelectionDAGBuilder::visitShuffleVector(
const User &
I) {
3679 if (
auto *SVI = dyn_cast<ShuffleVectorInst>(&
I))
3680 Mask = SVI->getShuffleMask();
3682 Mask = cast<ConstantExpr>(
I).getShuffleMask();
3688 if (
all_of(Mask, [](
int Elem) {
return Elem == 0; }) &&
3704 unsigned MaskNumElts =
Mask.size();
3706 if (SrcNumElts == MaskNumElts) {
3712 if (SrcNumElts < MaskNumElts) {
3716 if (MaskNumElts % SrcNumElts == 0) {
3720 unsigned NumConcat = MaskNumElts / SrcNumElts;
3721 bool IsConcat =
true;
3723 for (
unsigned i = 0; i != MaskNumElts; ++i) {
3729 if ((
Idx % SrcNumElts != (i % SrcNumElts)) ||
3730 (ConcatSrcs[i / SrcNumElts] >= 0 &&
3731 ConcatSrcs[i / SrcNumElts] != (
int)(
Idx / SrcNumElts))) {
3736 ConcatSrcs[i / SrcNumElts] =
Idx / SrcNumElts;
3743 for (
auto Src : ConcatSrcs) {
3756 unsigned PaddedMaskNumElts =
alignTo(MaskNumElts, SrcNumElts);
3757 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3774 for (
unsigned i = 0; i != MaskNumElts; ++i) {
3776 if (
Idx >= (
int)SrcNumElts)
3777 Idx -= SrcNumElts - PaddedMaskNumElts;
3785 if (MaskNumElts != PaddedMaskNumElts)
3793 if (SrcNumElts > MaskNumElts) {
3796 int StartIdx[2] = { -1, -1 };
3797 bool CanExtract =
true;
3798 for (
int Idx : Mask) {
3803 if (
Idx >= (
int)SrcNumElts) {
3812 if (NewStartIdx + MaskNumElts > SrcNumElts ||
3813 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3817 StartIdx[Input] = NewStartIdx;
3820 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3826 for (
unsigned Input = 0; Input < 2; ++Input) {
3827 SDValue &Src = Input == 0 ? Src1 : Src2;
3828 if (StartIdx[Input] < 0)
3838 for (
int &
Idx : MappedOps) {
3839 if (
Idx >= (
int)SrcNumElts)
3840 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3855 for (
int Idx : Mask) {
3861 SDValue &Src =
Idx < (int)SrcNumElts ? Src1 : Src2;
3862 if (
Idx >= (
int)SrcNumElts)
Idx -= SrcNumElts;
3876 const Value *Op0 =
I.getOperand(0);
3877 const Value *Op1 =
I.getOperand(1);
3878 Type *AggTy =
I.getType();
3880 bool IntoUndef = isa<UndefValue>(Op0);
3881 bool FromUndef = isa<UndefValue>(Op1);
3891 unsigned NumAggValues = AggValueVTs.
size();
3892 unsigned NumValValues = ValValueVTs.
size();
3896 if (!NumAggValues) {
3904 for (; i != LinearIndex; ++i)
3905 Values[i] = IntoUndef ?
DAG.
getUNDEF(AggValueVTs[i]) :
3910 for (; i != LinearIndex + NumValValues; ++i)
3911 Values[i] = FromUndef ?
DAG.
getUNDEF(AggValueVTs[i]) :
3915 for (; i != NumAggValues; ++i)
3916 Values[i] = IntoUndef ?
DAG.
getUNDEF(AggValueVTs[i]) :
3925 const Value *Op0 =
I.getOperand(0);
3927 Type *ValTy =
I.getType();
3928 bool OutOfUndef = isa<UndefValue>(Op0);
3936 unsigned NumValValues = ValValueVTs.
size();
3939 if (!NumValValues) {
3948 for (
unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3949 Values[i - LinearIndex] =
3958void SelectionDAGBuilder::visitGetElementPtr(
const User &
I) {
3959 Value *Op0 =
I.getOperand(0);
3969 bool IsVectorGEP =
I.getType()->isVectorTy();
3971 IsVectorGEP ? cast<VectorType>(
I.getType())->getElementCount()
3974 if (IsVectorGEP && !
N.getValueType().isVector()) {
3982 const Value *
Idx = GTI.getOperand();
3983 if (
StructType *StTy = GTI.getStructTypeOrNull()) {
3984 unsigned Field = cast<Constant>(
Idx)->getUniqueInteger().getZExtValue();
3993 if (int64_t(
Offset) >= 0 && cast<GEPOperator>(
I).isInBounds())
3994 Flags.setNoUnsignedWrap(
true);
4010 bool ElementScalable = ElementSize.
isScalable();
4014 const auto *
C = dyn_cast<Constant>(
Idx);
4015 if (
C && isa<VectorType>(
C->getType()))
4016 C =
C->getSplatValue();
4018 const auto *CI = dyn_cast_or_null<ConstantInt>(
C);
4019 if (CI && CI->isZero())
4021 if (CI && !ElementScalable) {
4035 Flags.setNoUnsignedWrap(
true);
4048 VectorElementCount);
4056 if (ElementScalable) {
4057 EVT VScaleTy =
N.getValueType().getScalarType();
4067 if (ElementMul != 1) {
4068 if (ElementMul.isPowerOf2()) {
4069 unsigned Amt = ElementMul.logBase2();
4071 N.getValueType(), IdxN,
4077 N.getValueType(), IdxN, Scale);
4083 N.getValueType(),
N, IdxN);
4094 if (PtrMemTy != PtrTy && !cast<GEPOperator>(
I).isInBounds())
4100void SelectionDAGBuilder::visitAlloca(
const AllocaInst &
I) {
4107 Type *Ty =
I.getAllocatedType();
4111 MaybeAlign Alignment = std::max(
DL.getPrefTypeAlign(Ty),
I.getAlign());
4133 if (*Alignment <= StackAlign)
4134 Alignment = std::nullopt;
4141 Flags.setNoUnsignedWrap(
true);
4151 DAG.
getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4167 if (!
I.hasMetadata(LLVMContext::MD_noundef))
4169 return I.getMetadata(LLVMContext::MD_range);
4172void SelectionDAGBuilder::visitLoad(
const LoadInst &
I) {
4174 return visitAtomicLoad(
I);